Repository: googleapis/python-spanner-django

Issue:
django: non-ascii SELECT alias crashes "Syntax error: Illegal input character"
This might be unsupported by Spanner.
e.g. `RawData.objects.filter(raw_data=b'\x00\x46  \xFE').extra(select={'föö': 1})`
Failure:
```
======================================================================
ERROR: test_query_encoding (backends.tests.LastExecutedQueryTest)
last_executed_query() returns a string.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 79, in next
    return six.next(self._wrapped)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 392, in __next__
    return self._next()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 561, in _next
    raise self
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Syntax error: Illegal input character \"\\303\" [at 1:16]\nSELECT (1) AS f\303\266\303\266, backends_rawdata.id, backends_rawdata.raw_data FROM back...\n               ^"
	debug_error_string = "{"created":"@1583849978.977754676","description":"Error received from peer ipv4:172.217.10.106:443","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"Syntax error: Illegal input character \"\\303\" [at 1:16]\nSELECT (1) AS f\303\266\303\266, backends_rawdata.id, backends_rawdata.raw_data FROM back...\n               ^","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/spanner-orm/spanner/dbapi/autocommit_on_cursor.py", line 91, in execute
    self.__handle_DQL(sql, args or None)
  File "/home/tim/code/spanner-orm/spanner/dbapi/autocommit_on_cursor.py", line 189, in __handle_DQL
    self.__itr = PeekIterator(self.__res)
  File "/home/tim/code/spanner-orm/spanner/dbapi/utils.py", line 27, in __init__
    head = next(itr_src)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 143, in __iter__
    self._consume_next()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 116, in _consume_next
    response = six.next(self._response_iterator)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/snapshot.py", line 45, in _restart_on_unavailable
    for item in iterator:
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 81, in next
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Syntax error: Illegal input character \"\\303\" [at 1:16]\nSELECT (1) AS f\303\266\303\266, backends_rawdata.id, backends_rawdata.raw_data FROM back...\n               ^

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/autocommit_on_cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Syntax error: Illegal input character \"\\303\" [at 1:16]\nSELECT (1) AS f\303\266\303\266, backends_rawdata.id, backends_rawdata.raw_data FROM back...\n               ^

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/django/tests/backends/tests.py", line 72, in test_query_encoding
    cursor = data.query.get_compiler('default').execute_sql(CURSOR)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 99, in execute
    return super().execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/tim/code/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/autocommit_on_cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Syntax error: Illegal input character \"\\303\" [at 1:16]\nSELECT (1) AS f\303\266\303\266, backends_rawdata.id, backends_rawdata.raw_data FROM back...\n               ^
```


Discussion:
Cloud Spanner supports Unicode, so if the characters are legal Unicode, then we shouldn't be crashing. Let me examine the emitted SQL.
I've seen the issued and failing SQL it is of the form
```sql
SELECT
    (1) AS föö, backends_rawdata.id, backends_rawdata.raw_data
FROM
   backends_rawdata
WHERE
   backends_rawdata.raw_data = %s
Args: (b'AEYgIP4=',)
```

but that fails because Spanner trips out on the umlauts in `föö`.

If we escape with backticks '`föö`', it works alright.

I wonder however, is the use of Unicode points as named column names a common thing?
This is the only usage in the Django test suite I came across (see https://github.com/django/django/commit/e9ef9776d186a3379cdbf47a73b14e89b74d0926 to see when it was added).

It may be common when writing code in languages that use such characters.
Thank you Tim for the response and references/advisory!

Given that Cloud Spanner by default advertises supporting Unicode, perhaps this
should go in. Am thinking we could write a character class for Unicode characters that are un-backticked, at least for DQL (SELECT), then replace them. Hope this doesn't slow things down a lot though, or we could pass in a setting that turns on Unicode backticking.

Resolving patch:
diff --git a/django_spanner/features.py b/django_spanner/features.py
--- a/django_spanner/features.py
+++ b/django_spanner/features.py
@@ -289,9 +289,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         # DatabaseIntrospection.get_relations() isn't implemented:
         # https://github.com/orijtech/django-spanner/issues/311
         'introspection.tests.IntrospectionTests.test_get_relations',
-        # Non-ascii SELECT alias crashes "Syntax error: Illegal input character"
-        # https://github.com/orijtech/django-spanner/issues/341
-        'backends.tests.LastExecutedQueryTest.test_query_encoding',
         # pyformat parameters not supported on INSERT:
         # https://github.com/orijtech/django-spanner/issues/343
         'backends.tests.BackendTestCase.test_cursor_execute_with_pyformat',
diff --git a/spanner/dbapi/parse_utils.py b/spanner/dbapi/parse_utils.py
--- a/spanner/dbapi/parse_utils.py
+++ b/spanner/dbapi/parse_utils.py
@@ -15,7 +15,7 @@
 from .exceptions import Error, ProgrammingError
 from .parser import parse_values
 from .types import DateStr, TimestampStr
-from .utils import unescape_percent_literals
+from .utils import sanitize_literals_for_upload
 
 STMT_DDL = 'DDL'
 STMT_NON_UPDATING = 'NON_UPDATING'
@@ -144,7 +144,7 @@ def parse_insert(insert_sql, params):
     after_values_sql = re_VALUES_TILL_END.findall(insert_sql)
     if not after_values_sql:
         # Case b)
-        insert_sql = unescape_percent_literals(insert_sql)
+        insert_sql = sanitize_literals_for_upload(insert_sql)
         return {
             'sql_params_list': [(insert_sql, None,)],
         }
@@ -156,7 +156,7 @@ def parse_insert(insert_sql, params):
         if pyformat_str_count > 0:
             raise ProgrammingError('no params yet there are %d "%s" tokens' % pyformat_str_count)
 
-        insert_sql = unescape_percent_literals(insert_sql)
+        insert_sql = sanitize_literals_for_upload(insert_sql)
         # Confirmed case of:
         # SQL: INSERT INTO T (a1, a2) VALUES (1, 2)
         # Params: None
@@ -180,7 +180,7 @@ def parse_insert(insert_sql, params):
         )
         values_pyformat = [str(arg) for arg in values.argv]
         rows_list = rows_for_insert_or_update(columns, params, values_pyformat)
-        insert_sql_preamble = unescape_percent_literals(insert_sql_preamble)
+        insert_sql_preamble = sanitize_literals_for_upload(insert_sql_preamble)
         for row in rows_list:
             sql_params_list.append((insert_sql_preamble, row,))
 
@@ -206,7 +206,7 @@ def parse_insert(insert_sql, params):
     sql_param_tuples = []
     for token_arg in values.argv:
         row_sql = before_values_sql + ' VALUES%s' % token_arg
-        row_sql = unescape_percent_literals(row_sql)
+        row_sql = sanitize_literals_for_upload(row_sql)
         row_params, params = tuple(params[0:len(token_arg)]), params[len(token_arg):]
         sql_param_tuples.append((row_sql, row_params,))
 
@@ -319,14 +319,14 @@ def sql_pyformat_args_to_spanner(sql, params):
         Params:   {'a0': 'a', 'a1': 23, 'a2': '888***'}
     """
     if not params:
-        return unescape_percent_literals(sql), params
+        return sanitize_literals_for_upload(sql), params
 
     found_pyformat_placeholders = re_PYFORMAT.findall(sql)
     params_is_dict = isinstance(params, dict)
 
     if params_is_dict:
         if not found_pyformat_placeholders:
-            return unescape_percent_literals(sql), params
+            return sanitize_literals_for_upload(sql), params
     else:
         n_params = len(params) if params else 0
         n_matches = len(found_pyformat_placeholders)
@@ -336,7 +336,7 @@ def sql_pyformat_args_to_spanner(sql, params):
                 'want %d args in %s' % (n_matches, found_pyformat_placeholders, n_params, params))
 
     if len(params) == 0:
-        return unescape_percent_literals(sql), params
+        return sanitize_literals_for_upload(sql), params
 
     named_args = {}
     # We've now got for example:
@@ -354,7 +354,7 @@ def sql_pyformat_args_to_spanner(sql, params):
         else:
             named_args[key] = cast_for_spanner(params[i])
 
-    return unescape_percent_literals(sql), named_args
+    return sanitize_literals_for_upload(sql), named_args
 
 
 def cast_for_spanner(param):
diff --git a/spanner/dbapi/utils.py b/spanner/dbapi/utils.py
--- a/spanner/dbapi/utils.py
+++ b/spanner/dbapi/utils.py
@@ -4,6 +4,7 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+import re
 from collections import namedtuple
 
 from google.cloud.spanner_v1 import param_types
@@ -75,10 +76,35 @@ def get_table_column_schema(spanner_db, table_name):
         return column_details
 
 
-def unescape_percent_literals(s):
+re_UNICODE_POINTS = re.compile(r'([^\s]*[\u0080-\uFFFF]+[^\s]*)')
+
+
+def backtick_unicode(sql):
+    matches = list(re_UNICODE_POINTS.finditer(sql))
+    if not matches:
+        return sql
+
+    segments = []
+
+    last_end = 0
+    for match in matches:
+        start, end = match.span()
+        if sql[start] != '`' and sql[end-1] != '`':
+            segments.append(sql[last_end:start] + '`' + sql[start:end] + '`')
+        else:
+            segments.append(sql[last_end:end])
+
+        last_end = end
+
+    return ''.join(segments)
+
+
+def sanitize_literals_for_upload(s):
     """
-    Convert %% (escaped percent literals) to %. Percent signs must be escaped when
+    Convert literals in s, to be fit for consumption by Cloud Spanner.
+    1. Convert %% (escaped percent literals) to %. Percent signs must be escaped when
     values like %s are used as SQL parameter placeholders but Spanner's query language
     uses placeholders like @a0 and doesn't expect percent signs to be escaped.
+    2. Quote words containing non-ASCII, with backticks, for example föö to `föö`.
     """
-    return s.replace('%%', '%')
+    return backtick_unicode(s.replace('%%', '%'))


Test patch:
diff --git a/tests/spanner/dbapi/test_parse_utils.py b/tests/spanner/dbapi/test_parse_utils.py
--- a/tests/spanner/dbapi/test_parse_utils.py
+++ b/tests/spanner/dbapi/test_parse_utils.py
@@ -15,6 +15,7 @@
     ensure_where_clause, escape_name, get_param_types, parse_insert,
     rows_for_insert_or_update, sql_pyformat_args_to_spanner, strip_backticks,
 )
+from spanner.dbapi.utils import backtick_unicode
 
 
 class ParseUtilsTests(TestCase):
@@ -411,3 +412,16 @@ def test_strip_backticks(self):
             with self.subTest(name=name):
                 got = strip_backticks(name)
                 self.assertEqual(got, want)
+
+    def test_backtick_unicode(self):
+        cases = [
+            ('SELECT (1) as foo WHERE 1=1', 'SELECT (1) as foo WHERE 1=1'),
+            ('SELECT (1) as föö', 'SELECT (1) as `föö`'),
+            ('SELECT (1) as `föö`', 'SELECT (1) as `föö`'),
+            ('SELECT (1) as `föö` `umläut', 'SELECT (1) as `föö` `umläut'),
+            ('SELECT (1) as `föö', 'SELECT (1) as `föö'),
+        ]
+        for sql, want in cases:
+            with self.subTest(sql=sql):
+                got = backtick_unicode(sql)
+                self.assertEqual(got, want)