Repository: googleapis/python-spanner-django

Issue:
dbapi: batch in a row CREATE TABLE, CREATE INDEX statements, which are currently individually long running operations
Currently, our tests are slowly for a variety of factors, but I noticed that statements like CREATE TABLE are created at the beginning one after a time and on average, a long running operation per CREATE TABLE takes 8 to 10 seconds so for example if we've got 25 of them, that'll take about ~4 minutes while if we sent them all as batches, that takes about 27 seconds and here is the demo

```python
from google.cloud import spanner_v1 as spanner

def main():
    client = spanner.Client()
    ins = client.instance('django-tests')
    if not ins.exists():
        ins.configuration_name = 'projects/appdev-soda-spanner-staging/instanceConfigs/regional-us-west2'
        _ = ins.create()

    db = ins.database('dbx')
    if not db.exists():
        db.create()

    stmts = [
        "CREATE TABLE django_content_type (id INT64 NOT NULL, app_label STRING(100) NOT NULL, model STRING(100) NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_permission (id INT64 NOT NULL, name STRING(255) NOT NULL, content_type_id INT64 NOT NULL, codename STRING(100) NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_group (id INT64 NOT NULL, name STRING(150) NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_group_permissions (id INT64 NOT NULL, group_id INT64 NOT NULL, permission_id INT64 NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_user (id INT64 NOT NULL, password STRING(128) NOT NULL, last_login TIMESTAMP, is_superuser BOOL NOT NULL, username STRING(150) NOT NULL, first_name STRING(30) NOT NULL, last_name STRING(150) NOT NULL, email STRING(254) NOT NULL, is_staff BOOL NOT NULL, is_active BOOL NOT NULL, date_joined TIMESTAMP NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_user_groups (id INT64 NOT NULL, user_id INT64 NOT NULL, group_id INT64 NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE auth_user_user_permissions (id INT64 NOT NULL, user_id INT64 NOT NULL, permission_id INT64 NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE django_session (session_key STRING(40) NOT NULL, session_data STRING(MAX) NOT NULL, expire_date TIMESTAMP NOT NULL) PRIMARY KEY(session_key)",
        "CREATE TABLE basic_article (id INT64 NOT NULL, headline STRING(100) NOT NULL, pub_date TIMESTAMP NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE basic_featuredarticle (id INT64 NOT NULL, article_id INT64 NOT NULL) PRIMARY KEY(id)",
        "CREATE TABLE basic_selfref (id INT64 NOT NULL, selfref_id INT64, article_id INT64) PRIMARY KEY(id)",
        "CREATE UNIQUE INDEX django_content_type_app_label_model_76bd3d3b_uniq ON django_content_type (app_label, model)",
        "CREATE UNIQUE INDEX auth_permission_content_type_id_codename_01ab375a_uniq ON auth_permission (content_type_id, codename)",
        "CREATE INDEX auth_permission_content_type_id_2f476e4b ON auth_permission (content_type_id)",
        "CREATE UNIQUE INDEX auth_group_permissions_group_id_permission_id_0cd325b0_uniq ON auth_group_permissions (group_id, permission_id)",
        "CREATE INDEX auth_group_permissions_group_id_b120cbf9 ON auth_group_permissions (group_id)",
        "CREATE INDEX auth_group_permissions_permission_id_84c5c92e ON auth_group_permissions (permission_id)",
        "CREATE UNIQUE INDEX auth_user_groups_user_id_group_id_94350c0c_uniq ON auth_user_groups (user_id, group_id)",
        "CREATE INDEX auth_user_groups_user_id_6a12ed8b ON auth_user_groups (user_id)",
        "CREATE INDEX auth_user_groups_group_id_97559544 ON auth_user_groups (group_id)",
        "CREATE UNIQUE INDEX auth_user_user_permissions_user_id_permission_id_14a6b632_uniq ON auth_user_user_permissions (user_id, permission_id)",
        "CREATE INDEX auth_user_user_permissions_user_id_a95ead1b ON auth_user_user_permissions (user_id)",
        "CREATE INDEX auth_user_user_permissions_permission_id_1fbb5f2c ON auth_user_user_permissions (permission_id)",
        "CREATE INDEX django_session_expire_date_a5c62663 ON django_session (expire_date)",
        "CREATE INDEX basic_selfref_selfref_id_fa437744 ON basic_selfref (selfref_id)",
        "CREATE INDEX basic_selfref_article_id_b9f2fc8c ON basic_selfref (article_id)",
    ]

    if os.environ.get('BATCHED'):
        lro = db.update_ddl(stmts)
        print(lro.result())
    else: # Unbatched
        for stmt in stmts:
            lro = db.update_ddl([stmt])
            print(lro.result())

    db.drop()

if __name__ == '__main__':
    main()
```

#### Unbatched
```shell
$ time python3 foo.py 

real	10m6.379s
user	0m0.612s
sys	0m0.174s
```
#### Batched
```
$ time BATCHED=true python3 foo.py 


real	0m27.215s
user	0m0.487s
sys	0m0.131s
```

which shows a 95.5% performance improvement.


Discussion:
Running the experiment in branch https://github.com/orijtech/spanner-orm/compare/batch-continguous-create_table+index-ops on Django doesn't show any difference because after every CREATE, Django requests for the created tables, which means any batching is rendered useless :(
What do you mean by "requests for the created tables"?
Django issues requests for the created tables by doing a table listing aka
```
            SELECT
              t.table_name
            FROM
              information_schema.tables AS t
            WHERE
              t.table_catalog = '' and t.table_schema = ''
Can you try to find exactly where that query is coming from? I see it https://github.com/django/django/blob/4161e35048d91fa84abf4dd2ebf64e04e0c37ca4/django/core/management/commands/migrate.py#L296 but I'm surprised there would be a need to execute it again after each table is created.

Resolving patch:
diff --git a/spanner/dbapi/connection.py b/spanner/dbapi/connection.py
--- a/spanner/dbapi/connection.py
+++ b/spanner/dbapi/connection.py
@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .cursor import OP_DELETE, OP_INSERT, OP_UPDATE, Cursor
+from .cursor import (
+    OP_CONN_CLOSE, OP_DDL, OP_DELETE, OP_INSERT, OP_UPDATE, Cursor,
+)
 from .exceptions import Error
 
 
@@ -21,6 +23,7 @@ def __init__(self, db_handle):
         self.__dbhandle = db_handle
         self.__closed = False
         self.__ops = []
+        self.__ddl_statements = []
 
     def __raise_if_already_closed(self):
         """
@@ -30,7 +33,7 @@ def __raise_if_already_closed(self):
             raise Error('attempting to use an already closed connection')
 
     def close(self):
-        self.commit()
+        self.commit(OP_CONN_CLOSE)
         self.__raise_if_already_closed()
         self.__dbhandle = None
         self.__closed = True
@@ -41,7 +44,9 @@ def __enter__(self):
     def __exit__(self, etype, value, traceback):
         return self.close()
 
-    def commit(self):
+    def commit(self, last_op=None):
+        self.__check_or_flush_update_ddl(last_op)
+
         if not self.__ops:
             return
 
@@ -65,7 +70,7 @@ def append_to_batch_stack(self, op, table, columns, values):
     def cursor(self):
         return Cursor(self)
 
-    def update_ddl(self, ddl_statements):
+    def __update_ddl(self, ddl_statements):
         """
         Runs the list of Data Definition Language (DDL) statements on the specified
         database. Note that each DDL statement MUST NOT contain a semicolon.
@@ -84,3 +89,22 @@ def read_snapshot(self):
 
     def in_transaction(self, fn, *args, **kwargs):
         return self.__dbhandle.run_in_transaction(fn, *args, **kwargs)
+
+    def handle_update_ddl(self, ddl_statement, prev_op=None):
+        if prev_op is None or prev_op != OP_DDL:
+            return self.__update_ddl([ddl_statement])
+        self.__ddl_statements.append(ddl_statement)
+
+    def __check_or_flush_update_ddl(self, last_op):
+        """
+        Run the batched DDL statements if last_op is a non-DDL statement.
+        """
+        if last_op == OP_DDL or last_op is None:
+            # Nothing to do here and we can keep on collecting
+            # DDL statements to later send in a batch.
+            return
+
+        if self.__ddl_statements:
+            ddl_statements = self.__ddl_statements
+            self.__ddl_statements = []
+            return self.__update_ddl(ddl_statements)
diff --git a/spanner/dbapi/cursor.py b/spanner/dbapi/cursor.py
--- a/spanner/dbapi/cursor.py
+++ b/spanner/dbapi/cursor.py
@@ -25,6 +25,9 @@
 OP_INSERT = 'insert'
 OP_UPDATE = 'update'
 OP_DELETE = 'delete'
+OP_DQL = 'dql'
+OP_DDL = 'ddl'
+OP_CONN_CLOSE = 'conn_close'
 
 
 class Cursor(object):
@@ -33,6 +36,7 @@ def __init__(self, db_handle=None):
         self.__res = None
         self.__row_count = _UNSET_COUNT
         self.__db_handle = db_handle
+        self.__last_op = None
 
         # arraysize is a readable and writable property mandated
         # by PEP-0249 https://www.python.org/dev/peps/pep-0249/#arraysize
@@ -58,7 +62,7 @@ def close(self):
         if self.__db_handle is None:
             return
 
-        self.__commit_preceding_batch()
+        self.__commit_preceding_batch(self.__last_op)
         self.__db_handle = None
 
     def execute(self, sql, args=None):
@@ -113,7 +117,7 @@ def execute(self, sql, args=None):
             raise OperationalError(e.details if hasattr(e, 'details') else e)
 
     def __handle_update(self, sql, params, param_types):
-        self.__commit_preceding_batch()
+        self.__commit_preceding_batch(OP_UPDATE)
         self.__db_handle.in_transaction(
             self.__do_execute_update,
             sql, params, param_types,
@@ -158,11 +162,13 @@ def __handle_insert(self, sql, params):
     def __execute_insert_no_params(self, transaction, sql):
         return transaction.execute_update(sql)
 
-    def __commit_preceding_batch(self):
-        self.__db_handle.commit()
+    def __commit_preceding_batch(self, op=None):
+        last_op = self.__last_op
+        self.__last_op = op
+        self.__db_handle.commit(last_op)
 
     def __handle_DQL(self, sql, params, param_types=None):
-        self.__commit_preceding_batch()
+        self.__commit_preceding_batch(OP_DQL)
 
         with self.__db_handle.read_snapshot() as snapshot:
             # Reference
@@ -207,7 +213,7 @@ def __next__(self):
         return next(self.__itr)
 
     def __iter__(self):
-        self.__commit_preceding_batch()
+        self.__commit_preceding_batch(OP_DQL)
 
         if self.__itr is None:
             raise ProgrammingError('no results to return')
@@ -257,11 +263,13 @@ def setinputsizes(sizes):
     def setoutputsize(size, column=None):
         raise ProgrammingError('Unimplemented')
 
-    def __handle_update_ddl(self, *ddl_statements):
+    def __handle_update_ddl(self, ddl_statement):
+        self.__commit_preceding_batch(OP_DDL)
+
         if not self.__db_handle:
             raise ProgrammingError('Trying to run an DDL update but no database handle')
 
-        return self.__db_handle.update_ddl(ddl_statements)
+        return self.__db_handle.handle_update_ddl(ddl_statement, self.__last_op)
 
 
 class Column:


Test patch:
diff --git a/django_test_suite.sh b/django_test_suite.sh
--- a/django_test_suite.sh
+++ b/django_test_suite.sh
@@ -1,11 +1,10 @@
 #!/bin/sh
 
-ORIGWD=$(pwd)
 # If no SPANNER_TEST_DB is set, generate a unique one
 # so that we can have multiple tests running without
 # conflicting which changes and constraints. We'll always
 # cleanup the created database.
-TEST_DBNAME=${SPANNER_TEST_DB:-testdb-$(date +%F%H%M%S)}
+TEST_DBNAME=${SPANNER_TEST_DB:-testdb-$(python3 -c 'import random; print(random.randint(1e3, 0x7fffffff))')}
 TEST_DBNAME_OTHER="$TEST_DBNAME-other"
 TEST_APPS=${DJANGO_TEST_APPS:-basic}
 INSTANCE_NAME=${SPANNER_TEST_INSTANCE:-django-tests}
@@ -19,6 +18,7 @@ REGION=${SPANNER_TEST_REGION:-regional-us-east4}
 INSTANCE_CONFIG="projects/$PROJECT/instanceConfigs/$REGION"
 
 DBs=($TEST_DBNAME $TEST_DBNAME_OTHER)
+ORIGWD=$(pwd)
 
 function create_db() {
     for DB in ${DBs[*]}