Repository: googleapis/python-spanner-django

Issue:
django: large delete fails with "Number of parameters in query exceeds the maximum allowed"
```
======================================================================
ERROR: test_large_delete (delete.tests.DeletionTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 690, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 592, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Number of parameters in query exceeds the maximum allowed limit of 950."
	debug_error_string = "{"created":"@1577497159.772387455","description":"Error received from peer ipv4:172.217.7.10:443","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"Number of parameters in query exceeds the maximum allowed limit of 950.","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 95, in execute
    self.__handle_update(sql, args or None)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 107, in __handle_update
    sql, params,
  File "/home/tim/code/spanner-orm/spanner/dbapi/connection.py", line 91, in in_transaction
    return self.__dbhandle.run_in_transaction(fn, *args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/database.py", line 444, in run_in_transaction
    return session.run_in_transaction(func, *args, **kw)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/session.py", line 300, in run_in_transaction
    return_value = func(txn, *args, **kw)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 114, in __do_execute_update
    res = transaction.execute_update(sql, params=params, param_types=get_param_types(params))
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 810, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/retry.py", line 277, in retry_wrapped_func
    on_error=on_error,
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/retry.py", line 182, in retry_target
    return target()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Number of parameters in query exceeds the maximum allowed limit of 950.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Number of parameters in query exceeds the maximum allowed limit of 950.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/django/tests/delete/tests.py", line 335, in test_large_delete
    self.assertNumQueries(queries, Avatar.objects.all().delete)
  File "/home/tim/code/django/django/test/testcases.py", line 1066, in assertNumQueries
    func(*args, **kwargs)
  File "/home/tim/code/django/django/db/models/query.py", line 711, in delete
    deleted, _rows_count = collector.delete()
  File "/home/tim/code/django/django/db/models/deletion.py", line 294, in delete
    count = qs._raw_delete(using=self.using)
  File "/home/tim/code/django/django/db/models/query.py", line 725, in _raw_delete
    return sql.DeleteQuery(self.model).delete_qs(self, using)
  File "/home/tim/code/django/django/db/models/sql/subqueries.py", line 75, in delete_qs
    cursor = self.get_compiler(using).execute_sql(CURSOR)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 99, in execute
    return super().execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/tim/code/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Number of parameters in query exceeds the maximum allowed limit of 950.

======================================================================
ERROR: test_large_delete_related (delete.tests.DeletionTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 690, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 592, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Number of parameters in query exceeds the maximum allowed limit of 950."
	debug_error_string = "{"created":"@1577497489.918971570","description":"Error received from peer ipv4:172.217.7.10:443","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"Number of parameters in query exceeds the maximum allowed limit of 950.","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 95, in execute
    self.__handle_update(sql, args or None)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 107, in __handle_update
    sql, params,
  File "/home/tim/code/spanner-orm/spanner/dbapi/connection.py", line 91, in in_transaction
    return self.__dbhandle.run_in_transaction(fn, *args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/database.py", line 444, in run_in_transaction
    return session.run_in_transaction(func, *args, **kw)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/session.py", line 300, in run_in_transaction
    return_value = func(txn, *args, **kw)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 114, in __do_execute_update
    res = transaction.execute_update(sql, params=params, param_types=get_param_types(params))
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 810, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/retry.py", line 277, in retry_wrapped_func
    on_error=on_error,
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/retry.py", line 182, in retry_target
    return target()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Number of parameters in query exceeds the maximum allowed limit of 950.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Number of parameters in query exceeds the maximum allowed limit of 950.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/django/tests/delete/tests.py", line 353, in test_large_delete_related
    self.assertNumQueries(expected_num_queries, s.delete)
  File "/home/tim/code/django/django/test/testcases.py", line 1066, in assertNumQueries
    func(*args, **kwargs)
  File "/home/tim/code/django/django/db/models/base.py", line 919, in delete
    return collector.delete()
  File "/home/tim/code/django/django/db/models/deletion.py", line 294, in delete
    count = qs._raw_delete(using=self.using)
  File "/home/tim/code/django/django/db/models/query.py", line 725, in _raw_delete
    return sql.DeleteQuery(self.model).delete_qs(self, using)
  File "/home/tim/code/django/django/db/models/sql/subqueries.py", line 75, in delete_qs
    cursor = self.get_compiler(using).execute_sql(CURSOR)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 99, in execute
    return super().execute(sql, params)
  File "/home/tim/code/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/tim/code/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/tim/code/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 99, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Number of parameters in query exceeds the maximum allowed limit of 950.
```


Discussion:

Resolving patch:
diff --git a/spanner/django/operations.py b/spanner/django/operations.py
--- a/spanner/django/operations.py
+++ b/spanner/django/operations.py
@@ -34,6 +34,9 @@ def quote_name(self, name):
             name = name.replace(' ', '_')
         return escape_name(name)
 
+    def bulk_batch_size(self, fields, objs):
+        return self.connection.features.max_query_params
+
     def bulk_insert_sql(self, fields, placeholder_rows):
         placeholder_rows_sql = (", ".join(row) for row in placeholder_rows)
         values_sql = ", ".join("(%s)" % sql for sql in placeholder_rows_sql)


Test patch: