Repository: googleapis/python-spanner-django

Issue:
dbapi: failed translation for `INSERT INTO...VALUES (%s, %s), (%s, %s)`, (v1, v2, v3, v3)
Noticed last night while going deeper with the django migrate command

```SQL
SQL:      INSERT INTO t (f1, f2, f3) VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s)
Params:   [1, 2, 3, 4, 5, 6, 7, 8, 9]
```

trips out but really it should parse out to
```python
{'table': 't', 'columns': ['f1', 'f2', 'f3'], params: [(1, 2, 3,), (4, 5, 6,), (7, 8, 9,)]}
```
and the problem comes because we only expected the form
```SQL
SQL:      INSERT INTO t (f1, f2, f3) VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s)
Params:   [1, 2, 3, 4, 5, 6, 7, 8, 9]
```


Discussion:

Resolving patch:
diff --git a/spanner/dbapi/cursor.py b/spanner/dbapi/cursor.py
--- a/spanner/dbapi/cursor.py
+++ b/spanner/dbapi/cursor.py
@@ -89,11 +89,7 @@ def execute(self, sql, args=None):
                 self.__session.run_in_transaction(
                     self.__do_execute_insert,
                     sql,
-                    # Insert statements' params are only passed as tuples or lists,
-                    # yet for do_execute_update, we've got to pass in list of list.
-                    # https://googleapis.dev/python/spanner/latest/transaction-api.html\
-                    #           #google.cloud.spanner_v1.transaction.Transaction.insert
-                    [args] if args else None,
+                    args if args else None,
                 )
             else:
                 self.__session.run_in_transaction(
@@ -127,7 +123,11 @@ def __do_execute_insert(self, transaction, sql, params):
         if params:
             # Case c)
             parts = parse_insert(sql)
-            columns, params = add_missing_id(parts.get('columns'), params, gen_rand_int64)
+            columns, params = add_missing_id(parts.get('columns'), params,
+                                             gen_rand_int64,
+                                             parts.get('values_pyformat'),
+                                             )
+
             return transaction.insert_or_update(
                 table=parts.get('table'),
                 columns=columns,
diff --git a/spanner/dbapi/parse_utils.py b/spanner/dbapi/parse_utils.py
--- a/spanner/dbapi/parse_utils.py
+++ b/spanner/dbapi/parse_utils.py
@@ -257,6 +257,7 @@ def classify_stmt(sql):
     re.DOTALL,
 )
 
+
 def parse_insert(insert_sql):
     match = re_INSERT.search(insert_sql)
     if not match:
@@ -275,23 +276,72 @@ def parse_insert(insert_sql):
     return parsed
 
 
-def add_missing_id(columns, params, gen_id):
+def add_missing_id(columns, params, gen_id, pyformat_args=None):
     if 'id' in columns:
         return (columns, params,)
 
+    new_params = copy.deepcopy(params)
+
+    if not pyformat_args:
+        # This is the case where we have for example:
+        # SQL:        'INSERT INTO t (f1, f2, f3)'
+        # Params A:   [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        # Params B:   [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        #
+        # We'll have to convert both params types into:
+        #           [(1, 2, 3,), (4, 5, 6,), (7, 8, 9,)]
+
+        contains_all_list_or_tuples = True
+        for param in params:
+            if not (isinstance(param, list) or isinstance(param, tuple)):
+                contains_all_list_or_tuples = False
+                break
+
+        if contains_all_list_or_tuples:
+            # The case with Params B: [(1, 2, 3,)]
+            for i, new_param in enumerate(new_params):
+                new_param += type(new_param)([gen_id()])
+                new_params[i] = new_param
+        else:
+            # The case with Params B: Params: [1, 2, 3]
+            new_params += type(new_params)([gen_id()])
+            # Insert statements' params are only passed as tuples or lists,
+            # yet for do_execute_update, we've got to pass in list of list.
+            # https://googleapis.dev/python/spanner/latest/transaction-api.html\
+            #           #google.cloud.spanner_v1.transaction.Transaction.insert
+            new_params = [tuple(new_params)]
+    else:
+        # This is the case where we have for example:
+        # SQL:      'INSERT INTO t (f1, f2, f3) VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s)'
+        # Params:   [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        #    which should become
+        # Columns:      (f1, f2, f3)
+        # new_params:   [(1, 2, 3,), (4, 5, 6,), (7, 8, 9,)]
+
+        # Sanity check: all the pyformat_values should have the exact same length.
+        first, rest = pyformat_args[0], pyformat_args[1:]
+        n_stride = first.count('%s')
+        for pyfmt_value in rest:
+            n = pyfmt_value.count('%s')
+            if n_stride != n:
+                raise Error('\nlen(`%s`)=%d\n!=\nlen(`%s`)=%d' % (
+                    first, n_stride, pyfmt_value, n))
+
+        # Now chop up the strides.
+        strides = []
+        for step in range(0, len(params), n_stride):
+            stride = tuple(params[step:step+n_stride:])
+            stride += (gen_id(),)
+            strides.append(stride)
+
+        new_params = strides
+
     # It is imperative that we copy and NOT mutate our original arguments as
     # references to them might be used elsewhere.
     new_columns = copy.deepcopy(columns)
-    new_params = copy.deepcopy(params)
-
-    # Otherwise auto insert 'id'.
+    # Now insert 'id' into all columns.
     new_columns += type(columns)(['id'])
 
-    # param is an iterable of iterables e.g. list of list.
-    for i, new_param in enumerate(new_params):
-        new_param += type(new_param)([gen_id()])
-        new_params[i] = new_param
-
     return (new_columns, new_params,)
 
 


Test patch:
diff --git a/tests/spanner/dbapi/test_parse_utils.py b/tests/spanner/dbapi/test_parse_utils.py
--- a/tests/spanner/dbapi/test_parse_utils.py
+++ b/tests/spanner/dbapi/test_parse_utils.py
@@ -207,7 +207,8 @@ def test_parse_insert(self):
             ),
             (
 
-                'INSERT INTO auth_permission (name, content_type_id, codename) VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s),(%s,      %s, %s)',
+                'INSERT INTO auth_permission (name, content_type_id, codename) '
+                'VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s),(%s,      %s, %s)',
                 {
                     'table': 'auth_permission',
                     'columns': ['name', 'content_type_id', 'codename'],
@@ -236,6 +237,7 @@ def next_id():
             (
                 ['id', 'app', 'name'],
                 [(5, 'ap', 'n',), (6, 'bp', 'm',)],
+                None,
                 (
                     ['id', 'app', 'name'],
                     [(5, 'ap', 'n',), (6, 'bp', 'm',)],
@@ -244,16 +246,48 @@ def next_id():
             (
                 ['app', 'name'],
                 [('ap', 'n',), ('bp', 'm',)],
+                None,
                 (
                     ['app', 'name', 'id'],
                     [('ap', 'n', 1,), ('bp', 'm', 2,)]
                 ),
             ),
+            (
+                ['app', 'name', 'fn'],
+                ['ap', 'n', 'f1', 'bp', 'm', 'f2', 'cp', 'o', 'f3'],
+                ['(%s, %s, %s)', '(%s, %s, %s)', '(%s, %s, %s)'],
+                (
+                    ['app', 'name', 'fn', 'id'],
+                    [('ap', 'n', 'f1', 3,), ('bp', 'm', 'f2', 4,), ('cp', 'o', 'f3', 5,)]
+                ),
+            ),
+            (
+                ['app', 'name', 'fn', 'ln'],
+                [('ap', 'n', (45, 'nested',), 'll',), ('bp', 'm', 'f2', 'mt',), ('fp', 'cp', 'o', 'f3',)],
+                None,
+                (
+                    ['app', 'name', 'fn', 'ln', 'id'],
+                    [
+                        ('ap', 'n', (45, 'nested',), 'll', 6,),
+                        ('bp', 'm', 'f2', 'mt', 7,),
+                        ('fp', 'cp', 'o', 'f3', 8,),
+                    ],
+                ),
+            ),
+            (
+                ['app', 'name', 'fn'],
+                ['ap', 'n', 'f1'],
+                None,
+                (
+                    ['app', 'name', 'fn', 'id'],
+                    [('ap', 'n', 'f1', 9,)]
+                ),
+            ),
         ]
-        for i, case in enumerate(cases):
-            columns, params, want = case
+
+        for i, (columns, params, pyformat_args, want) in enumerate(cases):
             with self.subTest(i=i):
-                got = add_missing_id(columns, params, next_id)
+                got = add_missing_id(columns, params, next_id, pyformat_args)
                 self.assertEqual(got, want)
 
     def test_sql_pyformat_args_to_spanner(self):