Repository: googleapis/python-spanner-django

Issue:
django: (i)contains, (i)startswith, and (i)endswith lookups with F expression crash
Support for QuerySets that use text lookups (e.g. startswith) with expressions on the right-hand side, e.g. `Author.objects.filter(name__startswith=Substr('alias', 1, 3))` isn't implemented.

```
======================================================================
ERROR: test_pattern_lookups_with_substr (lookup.tests.LookupTests) (lookup='startswith')
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/tim/code/django/tests/lookup/tests.py", line 919, in test_pattern_lookups_with_substr
    self.assertCountEqual(authors, result)
  File "/opt/python3.7.0/lib/python3.7/unittest/case.py", line 1182, in assertCountEqual
    first_seq, second_seq = list(first), list(second)
  File "/home/tim/code/django/django/db/models/query.py", line 274, in __iter__
    self._fetch_all()
  File "/home/tim/code/django/django/db/models/query.py", line 1242, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
  File "/home/tim/code/django/django/db/models/query.py", line 55, in __iter__
    results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1120, in execute_sql
    sql, params = self.as_sql()
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 489, in as_sql
    where, w_params = self.compile(self.where) if self.where is not None else ("", [])
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 405, in compile
    sql, params = node.as_sql(self, self.connection)
  File "/home/tim/code/django/django/db/models/sql/where.py", line 81, in as_sql
    sql, params = compiler.compile(child)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 403, in compile
    sql, params = vendor_impl(self, self.connection)
  File "/home/tim/code/spanner-orm/spanner/django/lookups.py", line 65, in startswith_endswith
    rhs_sql = self.get_rhs_op(connection, rhs_sql)
  File "/home/tim/code/django/django/db/models/lookups.py", line 395, in get_rhs_op
    pattern = connection.pattern_ops[self.lookup_name].format(connection.pattern_esc)
AttributeError: 'DatabaseWrapper' object has no attribute 'pattern_ops'
```


Discussion:

Resolving patch:
diff --git a/django_spanner/base.py b/django_spanner/base.py
--- a/django_spanner/base.py
+++ b/django_spanner/base.py
@@ -80,6 +80,23 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'iregex': 'REGEXP_CONTAINS(%s, %%%%s)',
     }
 
+    # pattern_esc is used to generate SQL pattern lookup clauses when the
+    # right-hand side of the lookup isn't a raw string (it might be an
+    # expression or the result of a bilateral transformation). In those cases,
+    # special characters for REGEXP_CONTAINS operators (e.g. \, *, _) must be
+    # escaped on database side.
+    pattern_esc = r'REPLACE(REPLACE(REPLACE({}, "\\", "\\\\"), "%%", r"\%%"), "_", r"\_")'
+    # These are all no-ops in favor of using REGEXP_CONTAINS in the customized
+    # lookups.
+    pattern_ops = {
+        'contains': '',
+        'icontains': '',
+        'startswith': '',
+        'istartswith': '',
+        'endswith': '',
+        'iendswith': '',
+    }
+
     Database = Database
     SchemaEditorClass = DatabaseSchemaEditor
     creation_class = DatabaseCreation
diff --git a/django_spanner/features.py b/django_spanner/features.py
--- a/django_spanner/features.py
+++ b/django_spanner/features.py
@@ -14,6 +14,8 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_introspect_big_integer_field = False
     can_introspect_duration_field = False
     closed_cursor_error_class = InterfaceError
+    # Spanner uses REGEXP_CONTAINS which is case-sensitive.
+    has_case_insensitive_like = False
     # https://cloud.google.com/spanner/quotas#query_limits
     max_query_params = 950
     supports_foreign_keys = False
@@ -134,11 +136,8 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         'model_fields.test_decimalfield.DecimalFieldTests.test_roundtrip_with_trailing_zeros',
         # No CHECK constraints in Spanner.
         'model_fields.test_integerfield.PositiveIntegerFieldTests.test_negative_values',
-        # 'DatabaseWrapper' object has no attribute 'pattern_ops'
-        # https://github.com/orijtech/spanner-orm/issues/178
-        'expressions.tests.BasicExpressionsTests.test_ticket_16731_startswith_lookup',
-        'expressions.tests.ExpressionsTests.test_insensitive_patterns_escape',
-        'expressions.tests.ExpressionsTests.test_patterns_escape',
+        # (i)contains, (i)startswith, and (i)endswith lookups with database
+        # function crash: https://github.com/googleapis/python-spanner-django/issues/409
         'lookup.tests.LookupTests.test_pattern_lookups_with_substr',
         # Spanner doesn't support the variance the standard deviation database
         # functions:
diff --git a/django_spanner/lookups.py b/django_spanner/lookups.py
--- a/django_spanner/lookups.py
+++ b/django_spanner/lookups.py
@@ -18,14 +18,23 @@ def contains(self, compiler, connection):
     rhs_sql, rhs_params = self.process_rhs(compiler, connection)
     params.extend(rhs_params)
     rhs_sql = self.get_rhs_op(connection, rhs_sql)
-    # Chop the leading and trailing percent signs that Django adds to the
-    # param since this isn't a LIKE query as Django expects.
-    params[0] = params[0][1:-1]
-    # Add the case insensitive flag for icontains.
-    if self.lookup_name.startswith('i'):
-        params[0] = '(?i)' + params[0]
-    # rhs_sql is REGEXP_CONTAINS(%s, %%s), and lhs_sql is the column name.
-    return rhs_sql % lhs_sql, params
+    is_icontains = self.lookup_name.startswith('i')
+    if params:
+        # Chop the leading and trailing percent signs that Django adds to the
+        # param since this isn't a LIKE query as Django expects.
+        params[0] = params[0][1:-1]
+        # Add the case insensitive flag for icontains.
+        if is_icontains:
+            params[0] = '(?i)' + params[0]
+        # rhs_sql is REGEXP_CONTAINS(%s, %%s), and lhs_sql is the column name.
+        return rhs_sql % lhs_sql, params
+    else:
+        # If no params, then rhs is the expression/column to use as the base of
+        # the regular expression.
+        rhs, _ = self.rhs.as_sql(compiler, connection)
+        if is_icontains:
+            rhs = "CONCAT('(?i)', " + rhs + ")"
+        return 'REGEXP_CONTAINS(%s, %s)' % (lhs_sql, connection.pattern_esc.format(rhs)), []
 
 
 def iexact(self, compiler, connection):
@@ -65,17 +74,34 @@ def startswith_endswith(self, compiler, connection):
     rhs_sql, rhs_params = self.process_rhs(compiler, connection)
     params.extend(rhs_params)
     rhs_sql = self.get_rhs_op(connection, rhs_sql)
+    is_startswith = 'startswith' in self.lookup_name
+    is_endswith = 'endswith' in self.lookup_name
+    is_insensitive = self.lookup_name.startswith('i')
     # Chop the leading (endswith) or trailing (startswith) percent sign that
     # Django adds to the param since this isn't a LIKE query as Django expects.
-    if 'endswith' in self.lookup_name:
-        params[0] = str(params[0][1:]) + '$'
+    if params:
+        if is_endswith:
+            params[0] = str(params[0][1:]) + '$'
+        else:
+            params[0] = '^' + str(params[0][:-1])
+            # Add the case insensitive flag for istartswith or iendswith.
+            if is_insensitive:
+                params[0] = '(?i)' + params[0]
+        # rhs_sql is REGEXP_CONTAINS(%s, %%s), and lhs_sql is the column name.
+        return rhs_sql % lhs_sql, params
     else:
-        params[0] = '^' + str(params[0][:-1])
-    # Add the case insentiive flag for istartswith or iendswith.
-    if self.lookup_name.startswith('i'):
-        params[0] = '(?i)' + params[0]
-    # rhs_sql is REGEXP_CONTAINS(%s, %%s), and lhs_sql is the column name.
-    return rhs_sql % lhs_sql, params
+        # If no params, then rhs is the expression/column to use as the base
+        # of the regular expression.
+        sql = "CONCAT('"
+        if is_startswith:
+            sql += '^'
+        if is_insensitive:
+            sql += '(?i)'
+        sql += "', " + self.rhs.as_sql(compiler, connection)[0]
+        if is_endswith:
+            sql += ", '$'"
+        sql += ")"
+        return 'REGEXP_CONTAINS(%s, %s)' % (lhs_sql, connection.pattern_esc.format(sql)), []
 
 
 def cast_param_to_float(self, compiler, connection):


Test patch: