Repository: googleapis/python-spanner-django

Issue:
django: adding NOT NULL column to existing table crashes
I wanted to put this integration and looked for already existing Django apps to test with the integration and I discovered https://github.com/realpython/materials/tree/master/rp-portfolio

On cd-ing into `rp-portfolio` and running
```shell
python3 manage.py migrate
```

I get this unhandled exception
```shell
Operations to perform:
  Apply all migrations: admin, auth, blog, contenttypes, projects, sessions
Running migrations:
  Applying contenttypes.0001_initial... OK
  Applying auth.0001_initial... OK
  Applying admin.0001_initial... OK
  Applying admin.0002_logentry_remove_auto_add... OK
  Applying admin.0003_logentry_add_action_flag_choices... OK
  Applying contenttypes.0002_remove_content_type_name... OK
  Applying auth.0002_alter_permission_name_max_length... OK
  Applying auth.0003_alter_user_email_max_length... OK
  Applying auth.0004_alter_user_username_opts... OK
  Applying auth.0005_alter_user_last_login_null... OK
  Applying auth.0006_require_contenttypes_0002... OK
  Applying auth.0007_alter_validators_add_error_messages... OK
  Applying auth.0008_alter_user_username_max_length... OK
  Applying auth.0009_alter_user_last_name_max_length... OK
  Applying auth.0010_alter_group_name_max_length... OK
  Applying auth.0011_update_proxy_permissions... OK
  Applying blog.0001_initial...Traceback (most recent call last):
  File "manage.py", line 17, in <module>
    execute_from_command_line(sys.argv)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/__init__.py", line 381, in execute_from_command_line
    utility.execute()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/__init__.py", line 375, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/base.py", line 323, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/base.py", line 364, in execute
    output = self.handle(*args, **options)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/base.py", line 83, in wrapped
    res = handle_func(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/core/management/commands/migrate.py", line 234, in handle
    fake_initial=fake_initial,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/migrations/executor.py", line 117, in migrate
    state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards
    state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/migrations/executor.py", line 245, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/migrations/migration.py", line 124, in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/migrations/operations/fields.py", line 112, in database_forwards
    field,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/base/schema.py", line 447, in add_field
    self.execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/base/schema.py", line 137, in execute
    cursor.execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/utils.py", line 99, in execute
    return super().execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 83, in execute
    self.__do_update_ddl(sql)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 241, in __do_update_ddl
    return self.__db_handle.update_ddl(ddl_statements)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/connection.py", line 69, in update_ddl
    return lro.result()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/future/polling.py", line 127, in result
    raise self._exception
google.api_core.exceptions.GoogleAPICallError: None Cannot add NOT NULL column blog_comment.post_id to existing table blog_comment.
```

and Cloud Spanner defines that it doesn't accept update operations that set NOT NULL to existent columns as per https://cloud.google.com/spanner/docs/schema-updates#performance
and in particular
![Screen Shot 2019-11-26 at 9 56 34 PM](https://user-images.githubusercontent.com/4898263/69697559-a1454900-1097-11ea-9e86-b5694d98779e.png)

if perhaps we can detect such operations in our SchemaEditor, we can make the change there. 
django: creating an index with a descending column fails
Creating an index with a descending column, e.g. `Index(fields=['-name’], name='idx')` generates SQL which doesn't work on Cloud Spanner: `CREATE INDEX "idx" ON "schema_author" ("name"DESC)` (lack of a space before DESC). This is [fixed in Django 3.1](https://code.djangoproject.com/ticket/30961). We may want to include a workaround in the adapter for Django 2.2 and 3.0.



Discussion:
False alarm, as I was just missing the proper migration steps and I fixed that by updating the website to document how to avoid such problems.
I've also seen this exception, I'd think it's a valid issue.
I encountered it because I didn't follow the right instructions and anyways we also can't add a NOT NULL to Cloud Spanner so basically the fix for adding a column will be to create a new table, move all the data there, delete the old/bad table, recreate the table with the new name and then move the data there, which is quite the involved step.
Would you recommennd we keep it open or nah?
It's still an issue (as discussed in https://github.com/orijtech/spanner-orm/issues/91) but the exception isn't surfacing because of #90.
I think we can add the column as null, then run `ALTER COLUMN column_name NOT NULL` afterward ([stackoverflow](https://stackoverflow.com/questions/42284232/adding-not-null-constraint-to-cloud-spanner-table)).
Just encountered this issue while running https://healthchecks.io/ https://github.com/healthchecks/healthchecks/ with spanner.django
```shell
  File "/Users/emmanuelodeke/Desktop/django-apps/healthchecks-webapp/hc-venv/lib/python3.7/site-packages/spanner/dbapi/autocommit_on_connection.py", line 63, in 
__handle_update_ddl
    return self.__dbhandle.update_ddl(ddl_statements).result()
  File "/Users/emmanuelodeke/Desktop/django-apps/healthchecks-webapp/hc-
venv/lib/python3.7/site-packages/google/api_core/future/polling.py", line 127, in result
    raise self._exception
google.api_core.exceptions.GoogleAPICallError: None Cannot add NOT NULL column 
accounts_profile.ping_log_limit to existing table accounts_profile.
```


Resolving patch:
diff --git a/django_spanner/features.py b/django_spanner/features.py
--- a/django_spanner/features.py
+++ b/django_spanner/features.py
@@ -132,10 +132,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         'db_functions.comparison.test_cast.CastTests.test_cast_to_decimal_field',
         'model_fields.test_decimalfield.DecimalFieldTests.test_fetch_from_db_without_float_rounding',
         'model_fields.test_decimalfield.DecimalFieldTests.test_roundtrip_with_trailing_zeros',
-        # No UNIQUE constraints in Spanner.
-        'auth_tests.test_basic.BasicTestCase.test_unicode_username',
-        'model_fields.test_filefield.FileFieldTests.test_unique_when_same_filename',
-        'one_to_one.tests.OneToOneTests.test_multiple_o2o',
         # No CHECK constraints in Spanner.
         'model_fields.test_integerfield.PositiveIntegerFieldTests.test_negative_values',
         # 'DatabaseWrapper' object has no attribute 'pattern_ops'
@@ -227,4 +223,34 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         # duplicate table raises GoogleAPICallError rather than DatabaseError:
         # https://github.com/orijtech/django-spanner/issues/344
         'backends.tests.BackendTestCase.test_duplicate_table_error',
+        # Spanner limitation: Cannot change type of column.
+        'schema.tests.SchemaTests.test_alter_auto_field_to_char_field',
+        'schema.tests.SchemaTests.test_alter_text_field_to_date_field',
+        'schema.tests.SchemaTests.test_alter_text_field_to_datetime_field',
+        'schema.tests.SchemaTests.test_alter_text_field_to_time_field',
+        # Spanner limitation: Cannot rename tables and columns.
+        'schema.tests.SchemaTests.test_alter_db_table_case',
+        'schema.tests.SchemaTests.test_alter_pk_with_self_referential_field',
+        'schema.tests.SchemaTests.test_rename',
+        'schema.tests.SchemaTests.test_db_table',
+        'schema.tests.SchemaTests.test_m2m_rename_field_in_target_model',
+        'schema.tests.SchemaTests.test_m2m_repoint',
+        'schema.tests.SchemaTests.test_m2m_repoint_custom',
+        'schema.tests.SchemaTests.test_m2m_repoint_inherited',
+        'schema.tests.SchemaTests.test_rename_column_renames_deferred_sql_references',
+        'schema.tests.SchemaTests.test_rename_keep_null_status',
+        'schema.tests.SchemaTests.test_rename_referenced_field',
+        'schema.tests.SchemaTests.test_rename_table_renames_deferred_sql_references',
+        'schema.tests.SchemaTests.test_referenced_field_without_constraint_rename_inside_atomic_block',
+        'schema.tests.SchemaTests.test_referenced_table_without_constraint_rename_inside_atomic_block',
+        'schema.tests.SchemaTests.test_unique_name_quoting',
+        # Spanner limitation: Cannot change a field to a primary key.
+        'schema.tests.SchemaTests.test_alter_not_unique_field_to_primary_key',
+        # Spanner limitation: Cannot drop column in primary key.
+        'schema.tests.SchemaTests.test_primary_key',
+        # Spanner limitation:  Cannot remove a column from the primary key.
+        'schema.tests.SchemaTests.test_alter_int_pk_to_int_unique',
+        # This test doesn't flush the database properly:
+        # https://code.djangoproject.com/ticket/31398
+        'multiple_database.tests.AuthTestCase',
     )
diff --git a/django_spanner/schema.py b/django_spanner/schema.py
--- a/django_spanner/schema.py
+++ b/django_spanner/schema.py
@@ -14,7 +14,8 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
     # Spanner doesn't support partial indexes. This string omits the
     # %(condition)s placeholder so that partial indexes are ignored.
     sql_create_index = "CREATE INDEX %(name)s ON %(table)s%(using)s (%(columns)s)%(extra)s"
-    sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
+    sql_create_unique = "CREATE UNIQUE NULL_FILTERED INDEX %(name)s ON %(table)s (%(columns)s)"
+    sql_delete_unique = "DROP INDEX %(name)s"
 
     # Cloud Spanner requires when changing if a column is NULLABLE,
     # that it should get redefined with its type and size.
@@ -63,6 +64,10 @@ def create_model(self, model):
                 self.quote_name(field.column),
                 definition,
             ))
+            # Create a unique constraint separately because Spanner doesn't
+            # allow them inline on a column.
+            if field.unique and not field.primary_key:
+                self.deferred_sql.append(self._create_unique_sql(model, [field.column]))
 
         # Add any unique_togethers (always deferred, as some fields might be
         # created afterwards, like geometry fields with some backends)
@@ -91,7 +96,63 @@ def create_model(self, model):
             if field.remote_field.through._meta.auto_created:
                 self.create_model(field.remote_field.through)
 
-    def column_sql(self, model, field, include_default=False):
+    def delete_model(self, model):
+        # Spanner requires dropping all of a table's indexes before dropping
+        # the table.
+        index_names = self._constraint_names(model, index=True, primary_key=False)
+        for index_name in index_names:
+            self.execute(self._delete_index_sql(model, index_name))
+        super().delete_model(model)
+
+    def add_field(self, model, field):
+        # Special-case implicit M2M tables
+        if field.many_to_many and field.remote_field.through._meta.auto_created:
+            return self.create_model(field.remote_field.through)
+        # Get the column's definition
+        definition, params = self.column_sql(model, field, exclude_not_null=True)
+        # It might not actually have a column behind it
+        if definition is None:
+            return
+        # Check constraints can go on the column SQL here
+        db_params = field.db_parameters(connection=self.connection)
+        if db_params['check']:
+            definition += " " + self.sql_check_constraint % db_params
+        # Build the SQL and run it
+        sql = self.sql_create_column % {
+            "table": self.quote_name(model._meta.db_table),
+            "column": self.quote_name(field.column),
+            "definition": definition,
+        }
+        self.execute(sql, params)
+        # Set defaults values on existing rows. (Django usually uses-database
+        # defaults for this but Spanner doesn't support them.)
+        effective_default = self.effective_default(field)
+        if effective_default is not None:
+            self.execute('UPDATE %(table)s SET %(column)s=%(value)r' % {
+                "table": self.quote_name(model._meta.db_table),
+                "column": self.quote_name(field.column),
+                "value": effective_default,
+            })
+        # Spanner doesn't support adding NOT NULL columns to existing tables.
+        if not field.null:
+            self.execute(self.sql_alter_column % {
+                "table": self.quote_name(model._meta.db_table),
+                "changes": self.sql_alter_column_not_null % {
+                    'column': self.quote_name(field.column),
+                    'type':  db_params['type'],
+                },
+            })
+        # Add an index, if required
+        self.deferred_sql.extend(self._field_indexes_sql(model, field))
+        # Create a unique constraint separately because Spanner doesn't allow
+        # them inline on a column.
+        if field.unique and not field.primary_key:
+            self.deferred_sql.append(self._create_unique_sql(model, [field.column]))
+        # Add any FK constraints later
+        if field.remote_field and self.connection.features.supports_foreign_keys and field.db_constraint:
+            self.deferred_sql.append(self._create_fk_sql(model, field, "_fk_%(to_table)s_%(to_column)s"))
+
+    def column_sql(self, model, field, include_default=False, exclude_not_null=False):
         """
         Take a field and return its column definition.
         The field must already have had set_attributes_from_name() called.
@@ -110,7 +171,7 @@ def column_sql(self, model, field, include_default=False):
         if (field.empty_strings_allowed and not field.primary_key and
                 self.connection.features.interprets_empty_strings_as_nulls):
             null = True
-        if not null:
+        if not null and not exclude_not_null:
             sql += " NOT NULL"
         # Optionally add the tablespace if it's an implicitly indexed column
         tablespace = field.db_tablespace or model._meta.db_tablespace
@@ -119,11 +180,36 @@ def column_sql(self, model, field, include_default=False):
         # Return the sql
         return sql, params
 
+    def add_index(self, model, index):
+        # Work around a bug in Django where a space isn't inserting before
+        # DESC: https://code.djangoproject.com/ticket/30961
+        # This method can be removed in Django 3.1.
+        index.fields_orders = [
+            (field_name, ' DESC' if order == 'DESC' else '')
+            for field_name, order in index.fields_orders
+        ]
+        super().add_index(model, index)
+
     def quote_value(self, value):
         # TODO: a real implementation:
         # https://github.com/orijtech/django-spanner/issues/227
         return str(value)
 
+    def _alter_column_type_sql(self, model, old_field, new_field, new_type):
+        # Spanner needs to use sql_alter_column_not_null if the field is
+        # NOT NULL, otherwise the constraint is dropped.
+        sql = self.sql_alter_column_type if new_field.null else self.sql_alter_column_not_null
+        return (
+            (
+                sql % {
+                    "column": self.quote_name(new_field.column),
+                    "type": new_type,
+                },
+                [],
+            ),
+            [],
+        )
+
     def _check_sql(self, name, check):
         # Spanner doesn't support CHECK constraints.
         return None


Test patch: