Repository: googleapis/python-spanner-django

Issue:
django: manage test database creation and destruction
Currently `django_test_suite.sh` manages test databases. It should instead be handled in `spanner.django.creation`.

Trying to execute `CREATE DATABASE` fails with the error: "CREATE DATABASE can be used only once." We'll have to use `google.cloud.spanner_v1` instead.


Discussion:

Resolving patch:
diff --git a/spanner/dbapi/__init__.py b/spanner/dbapi/__init__.py
--- a/spanner/dbapi/__init__.py
+++ b/spanner/dbapi/__init__.py
@@ -39,26 +39,14 @@
 threadsafety = 2  # Threads may share the module and connections but not cursors.
 
 
-def connect(spanner_url, credentials_uri=None):
-    """Connects to Cloud Spanner.
+def connect(project_name=None, instance_name=None, db_name=None, credentials_uri=None):
+    """
+    Connect to Cloud Spanner.
 
     Args:
-        spanner_url: A string specifying how to connect to Cloud Spanner.
-                     It will be of the format:
-                        cloudspanner:[//host[:port]]/projects/<project-id>/instances/<instance-id>/databases/<database-name>[?property-name=property-value[;]]*
-
-                     For example:
-                        * cloudspanner:/projects/foo/instances/bar/databases/db
-                        * cloudspanner://spanner.googleapis.com/projects/foo/instances/bar/databases/db
-                        * cloudspanner://spanner.googleapis.com:443/projects/foo/instances/bar/databases/db
-
-                    If an instance doesn't yet exist, the property
-                            "instance_config"
-                    must be set and match the format.
-                                ^projects/project-id/instanceConfigs/region$
-                    for example:
-                                projects/odeke-sandbox/instanceConfigs/regional-us-west2
-
+        project_name: A project that already exists.
+        instance_name: An instance that already exists.
+        db_name: A database that already exists.
         credentials_uri: An optional string specifying where to retrieve the service
                          account JSON for the credentials to connect to Cloud Spanner.
 
@@ -68,37 +56,27 @@ def connect(spanner_url, credentials_uri=None):
     Raises:
         Error if it encounters any unexpected inputs.
     """
-
-    conn_params = parse_spanner_url(spanner_url)
-
-    kwargs = dict(
-        project=conn_params.get('project_id'),
-        client_info=google_client_info(),
-    )
-
-    # Pre-requisite are the database and instance names.
-    db_name = conn_params.get('database')
     if not db_name:
-        raise Error("expected 'database'")
-    instance_name = conn_params.get('instance')
+        raise Error("'db_name' is required.")
     if not instance_name:
-        raise Error("expected 'instance'")
-
-    credentials_uri = conn_params.get('credentials_uri')
-    client = None
+        raise Error("'instance_name' is required.")
 
+    client_kwargs = {
+        'project': project_name,
+        'client_info': google_client_info(),
+    }
     if credentials_uri:
-        client = spanner.Client.from_service_account_json(credentials_uri, **kwargs)
+        client = spanner.Client.from_service_account_json(credentials_uri, **client_kwargs)
     else:
-        client = spanner.Client(**kwargs)
+        client = spanner.Client(**client_kwargs)
 
     instance = client.instance(instance_name)
     if not instance.exists():
-        raise ProgrammingError("instance '%s' does not yet exist" % instance_name)
+        raise ProgrammingError("instance '%s' does not exist." % instance_name)
 
     db = instance.database(db_name)
     if not db.exists():
-        raise ProgrammingError("database '%s' does not yet exist" % db_name)
+        raise ProgrammingError("database '%s' does not exist." % db_name)
 
     return Connection(db)
 
diff --git a/spanner/django/base.py b/spanner/django/base.py
--- a/spanner/django/base.py
+++ b/spanner/django/base.py
@@ -14,6 +14,7 @@
 
 import spanner.dbapi as Database
 from django.db.backends.base.base import BaseDatabaseWrapper
+from google.cloud import spanner_v1 as spanner
 
 from .client import DatabaseClient
 from .creation import DatabaseCreation
@@ -97,8 +98,21 @@ class DatabaseWrapper(BaseDatabaseWrapper):
     ops_class = DatabaseOperations
     client_class = DatabaseClient
 
+    @property
+    def instance(self):
+        return spanner.Client().instance(self.settings_dict['INSTANCE'])
+
+    @property
+    def _nodb_connection(self):
+        raise NotImplementedError('Spanner does not have a "no db" connection.')
+
     def get_connection_params(self):
-        return {'spanner_url': self.settings_dict['SPANNER_URL']}
+        return {
+            'project_name': self.settings_dict['PROJECT'],
+            'instance_name': self.settings_dict['INSTANCE'],
+            'db_name': self.settings_dict['NAME'],
+            **self.settings_dict['OPTIONS'],
+        }
 
     def get_new_connection(self, conn_params):
         return Database.connect(**conn_params)
diff --git a/spanner/django/creation.py b/spanner/django/creation.py
--- a/spanner/django/creation.py
+++ b/spanner/django/creation.py
@@ -1,4 +1,5 @@
 import os
+import sys
 from unittest import skip
 
 from django.conf import settings
@@ -25,6 +26,44 @@ def create_test_db(self, *args, **kwargs):
             self.mark_skips()
         super().create_test_db(*args, **kwargs)
 
+    def _create_test_db(self, verbosity, autoclobber, keepdb=False):
+        # Mostly copied from the base class but removes usage of
+        # _nodb_connection since Spanner doesn't have or need one.
+        test_database_name = self._get_test_db_name()
+        # Don't quote the test database name because google.cloud.spanner_v1
+        # does it.
+        test_db_params = {'dbname': test_database_name}
+        # Create the test database.
+        try:
+            self._execute_create_test_db(None, test_db_params, keepdb)
+        except Exception as e:
+            # If the db should be kept, then no need to do any of the below,
+            # just return and skip it all.
+            if keepdb:
+                return test_database_name
+            self.log('Got an error creating the test database: %s' % e)
+            if not autoclobber:
+                confirm = input(
+                    "Type 'yes' if you would like to try deleting the test "
+                    "database '%s', or 'no' to cancel: " % test_database_name)
+            if autoclobber or confirm == 'yes':
+                try:
+                    if verbosity >= 1:
+                        self.log('Destroying old test database for alias %s...' % (
+                            self._get_database_display_str(verbosity, test_database_name),
+                        ))
+                    self._destroy_test_db(test_database_name, verbosity)
+                    self._execute_create_test_db(None, test_db_params, keepdb)
+                except Exception as e:
+                    self.log('Got an error recreating the test database: %s' % e)
+                    sys.exit(2)
+            else:
+                self.log('Tests cancelled.')
+                sys.exit(1)
+        return test_database_name
+
+    def _execute_create_test_db(self, cursor, parameters, keepdb=False):
+        self.connection.instance.database(parameters['dbname']).create()
+
     def _destroy_test_db(self, test_database_name, verbosity):
-        # Cloud Spanner doesn't support dropping a database.
-        pass
+        self.connection.instance.database(test_database_name).drop()


Test patch:
diff --git a/django_test_suite.sh b/django_test_suite.sh
--- a/django_test_suite.sh
+++ b/django_test_suite.sh
@@ -1,5 +1,8 @@
 #!/bin/sh
 
+# exit when any command fails
+set -e
+
 # If no SPANNER_TEST_DB is set, generate a unique one
 # so that we can have multiple tests running without
 # conflicting which changes and constraints. We'll always
@@ -7,53 +10,15 @@
 TEST_DBNAME=${SPANNER_TEST_DB:-testdb-$(python3 -c 'import random; print(random.randint(1e3, 0x7fffffff))')}
 TEST_DBNAME_OTHER="$TEST_DBNAME-other"
 TEST_APPS=${DJANGO_TEST_APPS:-basic}
-INSTANCE_NAME=${SPANNER_TEST_INSTANCE:-django-tests}
+INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests}
 PROJECT=${SPANNER_TEST_PROJECT:-appdev-soda-spanner-staging}
 SETTINGS_FILE="$TEST_DBNAME-settings"
-DROPDB_ON_EXIT=${SPANNER_DROP_DB_ON_EXIT:-true}
-
-# Use "regional-us-east4" to reduce latency to Travis CI, as per:
-# https://devops.stackexchange.com/questions/2488/geographically-where-are-the-travis-ci-jobs-run
-REGION=${SPANNER_TEST_REGION:-regional-us-east4}
-INSTANCE_CONFIG="projects/$PROJECT/instanceConfigs/$REGION"
-
-DBs=($TEST_DBNAME $TEST_DBNAME_OTHER)
-ORIGWD=$(pwd)
-
-function create_db() {
-    for DB in ${DBs[*]}
-    do
-        echo "
-from google.cloud import spanner_v1 as sp
-ins = sp.Client(project='$PROJECT').instance('$INSTANCE_NAME')
-if not ins.exists():
-    ins.configuration_name = '$INSTANCE_CONFIG'
-    lro = ins.create()
-    _ = lro.result()
-    print('Created instance: $INSTANCE_NAME at $INSTANCE_CONFIG')
-
-db = ins.database('$DB')
-if not db.exists():
-    lro = db.create()
-    _ = lro.result()
-    print('Created database: $DB')
-else:
-    print('$DB already exists')
-" | python3 -
-        code=$?
-        if [[ $code -ne 0 ]]
-        then
-            return $code
-        fi
-    done
-}
 
 function checkout_django() {
     mkdir -p django_tests && cd django_tests
     git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git
     cd django && pip3 install -e .
     pip3 install -r tests/requirements/py3.txt
-    return $?
 }
 
 function create_settings() {
@@ -61,17 +26,15 @@ function create_settings() {
 DATABASES = {
    'default': {
        'ENGINE': 'spanner.django',
-       'SPANNER_URL': "cloudspanner:/projects/$PROJECT/instances/$INSTANCE_NAME/databases/$TEST_DBNAME?instance_config=$INSTANCE_CONFIG",
-        'TEST': {
-            'NAME': "$TEST_DBNAME",
-        },
+       'PROJECT': "$PROJECT",
+       'INSTANCE': "$INSTANCE",
+       'NAME': "$TEST_DBNAME",
    },
    'other': {
        'ENGINE': 'spanner.django',
-       'SPANNER_URL': "cloudspanner:/projects/$PROJECT/instances/$INSTANCE_NAME/databases/$TEST_DBNAME_OTHER?instance_config=$INSTANCE_CONFIG",
-        'TEST': {
-            'NAME': "$TEST_DBNAME_OTHER",
-        },
+       'PROJECT': "$PROJECT",
+       'INSTANCE': "$INSTANCE",
+       'NAME': "$TEST_DBNAME_OTHER",
    },
 }
 SECRET_KEY = 'spanner_tests_secret_key'
@@ -81,59 +44,17 @@ PASSWORD_HASHERS = [
 !
 }
 
-function drop_db() {
-    for DB in ${DBs[*]}
-    do
-        echo "
-from google.cloud import spanner_v1 as sp
-ins = sp.Client(project='$PROJECT').instance('$INSTANCE_NAME')
-if not ins.exists():
-    print('Instance $INSTANCE_NAME does not exist anyways')
-else:
-    db = ins.database('$DB')
-    if db.exists():
-        db.drop()
-        print('Dropped $DB')
-" | python3 -
-         code=$?
-        if [[ $code -ne 0 ]]
-        then
-            return $code
-        fi
-    done
-}
-
 function run_django_tests() {
     cd tests
     create_settings
     echo -e "\033[32mRunning Django tests $TEST_APPS\033[00m"
-    # Using --keepdb here because Django's test suite tries to invoke `DROP DATABASE`
-    # using the Cursor if --keepdb isn't set, yet only the Spanner.Database instance
-    # can drop the database, which we do manually in function `drop_db`.
-    python3 runtests.py $TEST_APPS --verbosity=2 --keepdb --noinput --settings $SETTINGS_FILE
-    return $?
+    python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE
 }
 
 function install_spanner_django() {
     pip3 install .
-    return $?
-}
-
-function cleanup_and_exit() {
-    exitCode=$?
-    msg=$1
-    echo $msg
-    if [[ $DROPDB_ON_EXIT = "true" ]]
-    then
-        drop_db
-    fi
-    cd $ORIGWD
-    exit $exitCode
 }
 
-install_spanner_django || cleanup_and_exit
-create_db || cleanup_and_exit "INSTANCE: $INSTANCE_NAME DB: $TEST_DBNAME and $TEST_DBNAME_OTHER could not be created"
-checkout_django || cleanup_and_exit
-run_django_tests || cleanup_and_exit
-# Unconditionally clean up before exit.
-cleanup_and_exit
+install_spanner_django
+checkout_django
+run_django_tests