Repository: googleapis/python-spanner-django

Issue:
dbapi: handle Cursor.execute(SQL_with_%s+, params)
Given this SQL to execute
```python
cur.execute('SELECT django_content_type.id, django_content_type.app_label, django_content_type.model FROM django_content_type WHERE django_content_type.app_label = %s', ('admin',))
```

we currently get this panic
```shell
Traceback (most recent call last):
  File "conn.py", line 53, in <module>
    alt()
  File "conn.py", line 27, in alt
    cur.execute('SELECT django_content_type.id, django_content_type.app_label, django_content_type.model FROM django_content_type WHERE django_content_type.app_label = %s', ('admin',))
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 88, in execute
    param_types=param_types,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 148, in __do_execute_non_update
    res = self.__session.execute_sql(sql, *args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/session.py", line 232, in execute_sql
    sql, params, param_types, query_mode, retry=retry, timeout=timeout
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/snapshot.py", line 204, in execute_sql
    fields={key: _make_value_pb(value) for key, value in params.items()}
AttributeError: 'tuple' object has no attribute 'items'
```

and really Spanner wants the `@ named` args so we'll need to transform from
```SQL
SELECT
     django_content_type.id,
     django_content_type.app_label,
     django_content_type.model
 FROM
    django_content_type
WHERE
    django_content_type.app_label = %s,
    other = %s,
    next = %s

Params: ('admin', 'this', 99,)
```
to
```SQL
SELECT
     django_content_type.id,
     django_content_type.app_label,
     django_content_type.model
 FROM
    django_content_type
WHERE
    django_content_type.app_label = @arg1
    other = @arg2,
    next = @arg3

Params: {'arg1': 'admin', 'arg2': 'this',  'arg3': 99}
```


Discussion:

Resolving patch:
diff --git a/spanner/dbapi/cursor.py b/spanner/dbapi/cursor.py
--- a/spanner/dbapi/cursor.py
+++ b/spanner/dbapi/cursor.py
@@ -19,7 +19,7 @@
 from .exceptions import IntegrityError, OperationalError, ProgrammingError
 from .parse_utils import (
     STMT_DDL, STMT_INSERT, STMT_NON_UPDATING, add_missing_id, classify_stmt,
-    parse_insert,
+    parse_insert, sql_pyformat_args_to_spanner,
 )
 
 _UNSET_COUNT = -1
@@ -142,10 +142,11 @@ def __do_execute_insert(self, transaction, sql, params):
     def gen_uuid4(self):
         return str(uuid4())
 
-    def __do_execute_non_update(self, sql, *args, **kwargs):
+    def __do_execute_non_update(self, sql, params, **kwargs):
         # Reference
         #  https://googleapis.dev/python/spanner/latest/session-api.html#google.cloud.spanner_v1.session.Session.execute_sql
-        res = self.__session.execute_sql(sql, *args, **kwargs)
+        sql, params = sql_pyformat_args_to_spanner(sql, params)
+        res = self.__session.execute_sql(sql, params=params, **kwargs)
         if type(res) == int:
             self.__row_count = res
             self.__itr = None
diff --git a/spanner/dbapi/parse_utils.py b/spanner/dbapi/parse_utils.py
--- a/spanner/dbapi/parse_utils.py
+++ b/spanner/dbapi/parse_utils.py
@@ -275,3 +275,62 @@ def add_missing_id(columns, params, gen_id):
         new_params[i] = new_param
 
     return (new_columns, new_params,)
+
+
+re_PYFORMAT = re.compile(r'(%s|%\([^\(\)]+\)s)+', re.DOTALL)
+
+
+def sql_pyformat_args_to_spanner(sql, params):
+    """
+    Transform pyformat set SQL to named arguments for Cloud Spanner.
+    For example:
+        SQL:      'SELECT * from t where f1=%s, f2=%s, f3=%s'
+        Params:   ('a', 23, '888***')
+    becomes:
+        SQL:      'SELECT * from t where f1=@a0, f2=@a1, f3=@a2'
+        Params:   {'a0': 'a', 'a1': 23, 'a2': '888***'}
+
+    OR
+        SQL:      'SELECT * from t where f1=%(f1)s, f2=%(f2)s, f3=%(f3)s'
+        Params:   {'f1': 'a', 'f2': 23, 'f3': '888***', 'extra': 'aye')
+    becomes:
+        SQL:      'SELECT * from t where f1=@a0, f2=@a1, f3=@a2'
+        Params:   {'a0': 'a', 'a1': 23, 'a2': '888***'}
+    """
+    if not params:
+        return sql, params
+
+    found_pyformat_placeholders = re_PYFORMAT.findall(sql)
+    params_is_dict = isinstance(params, dict)
+
+    if params_is_dict:
+        if not found_pyformat_placeholders:
+            return sql, params
+    else:
+        n_params = len(params) if params else 0
+        n_matches = len(found_pyformat_placeholders)
+        if n_matches != n_params:
+            raise Error(
+                'pyformat_args mismatch\ngot %d args from %s\n'
+                'want %d args in %s' % (n_matches, found_pyformat_placeholders, n_params, params))
+
+    if len(params) == 0:
+        return sql, params
+
+    named_args = {}
+    # We've now got for example:
+    # Case a) Params is a non-dict
+    #   SQL:      'SELECT * from t where f1=%s, f2=%s, f3=%s'
+    #   Params:   ('a', 23, '888***')
+    # Case b) Params is a dict and the matches are %(value)s'
+    for i, pyfmt in enumerate(found_pyformat_placeholders):
+        key = 'a%d' % i
+        sql = sql.replace(pyfmt, '@'+key, 1)
+        if params_is_dict:
+            # The '%(key)s' case, so interpolate it.
+            resolved_value = pyfmt % params
+            named_args[key] = resolved_value
+        else:
+            named_args[key] = params[i]
+
+    return sql, named_args


Test patch:
diff --git a/tests/spanner/dbapi/test_parse_utils.py b/tests/spanner/dbapi/test_parse_utils.py
--- a/tests/spanner/dbapi/test_parse_utils.py
+++ b/tests/spanner/dbapi/test_parse_utils.py
@@ -18,7 +18,7 @@
 from spanner.dbapi.parse_utils import (
     STMT_DDL, STMT_NON_UPDATING, add_missing_id, classify_stmt,
     extract_connection_params, parse_insert, parse_spanner_url,
-    reINSTANCE_CONFIG, validate_instance_config,
+    reINSTANCE_CONFIG, sql_pyformat_args_to_spanner, validate_instance_config,
 )
 
 
@@ -245,3 +245,52 @@ def next_id():
             with self.subTest(i=i):
                 got = add_missing_id(columns, params, next_id)
                 self.assertEqual(got, want)
+
+    def test_sql_pyformat_args_to_spanner(self):
+        cases = [
+            (
+                ('SELECT * from t WHERE f1=%s, f2 = %s, f3=%s', (10, 'abc', 'y**$22l3f',)),
+                ('SELECT * from t WHERE f1=@a0, f2 = @a1, f3=@a2', {'a0': 10, 'a1': 'abc', 'a2': 'y**$22l3f'}),
+            ),
+            (
+                ('INSERT INTO t (f1, f2, f2) VALUES (%s, %s, %s)', ('app', 'name', 'applied',)),
+                ('INSERT INTO t (f1, f2, f2) VALUES (@a0, @a1, @a2)', {'a0': 'app', 'a1': 'name', 'a2': 'applied'}),
+            ),
+            (
+                (
+                    'INSERT INTO t (f1, f2, f2) VALUES (%(f1)s, %(f2)s, %(f3)s)',
+                    {'f1': 'app', 'f2': 'name', 'f3': 'applied'},
+                ),
+                (
+                    'INSERT INTO t (f1, f2, f2) VALUES (@a0, @a1, @a2)',
+                    {'a0': 'app', 'a1': 'name', 'a2': 'applied'},
+                ),
+            ),
+            (
+                # Intentionally using a dict with more keys than will be resolved.
+                ('SELECT * from t WHERE f1=%(f1)s', {'f1': 'app', 'f2': 'name'}),
+                ('SELECT * from t WHERE f1=@a0', {'a0': 'app'}),
+            ),
+            (
+                # No args to replace, we MUST return the original params dict
+                # since it might be useful to pass to the next user.
+                ('SELECT * from t WHERE id=10', {'f1': 'app', 'f2': 'name'}),
+                ('SELECT * from t WHERE id=10', {'f1': 'app', 'f2': 'name'}),
+            ),
+        ]
+        for ((sql_in, params), sql_want) in cases:
+            with self.subTest(sql=sql_in):
+                got_sql, got_named_args = sql_pyformat_args_to_spanner(sql_in, params)
+                want_sql, want_named_args = sql_want
+                self.assertEqual(got_sql, want_sql, 'SQL does not match')
+                self.assertEqual(got_named_args, want_named_args, 'Named args do not match')
+
+    def test_sql_pyformat_args_to_spanner_invalid(self):
+        cases = [
+            ('SELECT * from t WHERE f1=%s, f2 = %s, f3=%s, extra=%s', (10, 'abc', 'y**$22l3f',)),
+        ]
+        for sql, params in cases:
+            with self.subTest(sql=sql):
+                self.assertRaisesRegex(Error, 'pyformat_args mismatch',
+                                       lambda: sql_pyformat_args_to_spanner(sql, params),
+                                       )