Repository: googleapis/python-spanner-django

Issue:
django-test-suite: i18n failure, django.models.DecimalField should be inferred as FLOAT64
Currently there is a single failure when running the i18n test suite
- [ ] test_safestr (i18n.tests.TestModels)

where we also need to infer the type for FLOAT64 as we do in
https://github.com/orijtech/spanner-orm/blob/8d19a4aaafbe54f006b6c049a6647505e870c206/spanner/dbapi/parse_utils.py#L408-L424

we should add a clause for FLOAT64 too.

but also reeported by a bunch of errors in the Django test suite for example i18n fails with
```shell
======================================================================
ERROR: test_safestr (i18n.tests.TestModels)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 565, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 467, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Invalid value for bind parameter a2: Expected FLOAT64."
	debug_error_string = "{"created":"@1575451932.164210000","description":"Error received from peer ipv4:172.217.4.170:443","file":"src/core/lib/surface/call.cc","file_line":1046,"grpc_message":"Invalid value for bind parameter a2: Expected FLOAT64.","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 100, in execute
    param_types=param_types,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/session.py", line 299, in run_in_transaction
    return_value = func(txn, *args, **kw)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 122, in __do_execute_update
    res = transaction.execute_update(sql, params=params, param_types=param_types)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 812, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/retry.py", line 277, in retry_wrapped_func
    on_error=on_error,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/retry.py", line 182, in retry_target
    return target()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Invalid value for bind parameter a2: Expected FLOAT64.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 107, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Invalid value for bind parameter a2: Expected FLOAT64.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/tests/i18n/tests.py", line 1536, in test_safestr
    c.save()
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/base.py", line 741, in save
    force_update=force_update, update_fields=update_fields)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/base.py", line 779, in save_base
    force_update, using, update_fields,
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/base.py", line 851, in _save_table
    forced_update)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/base.py", line 900, in _do_update
    return filtered._update(values) > 0
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/query.py", line 760, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1429, in execute_sql
    cursor = super().execute_sql(result_type)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1100, in execute_sql
    cursor.execute(sql, params)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/Users/emmanuelodeke/Desktop/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 107, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Invalid value for bind parameter a2: Expected FLOAT64.

----------------------------------------------------------------------
Ran 198 tests in 31.111s

FAILED (errors=1, skipped=63)
Preserving test database for alias 'default' ('db1')...
```


Discussion:
The error occurs because a models.DecimalField is being serialized as '12.00' yet we haven't inferred its type
![Screen Shot 2019-12-04 at 5 40 34 PM](https://user-images.githubusercontent.com/4898263/70196343-4fb34480-16bd-11ea-8e59-622bdf6c7898.png)


and the statement that gets executed is:

```python
SQL: UPDATE i18n_company SET name = @a0, date_added = @a1,
           cents_paid = @a2, products_delivered = @a3 WHERE i18n_company.id = @a4

params: {'a0': 'Iñtërnâtiônàlizætiøn1', 'a1': '1799-01-31T23:59:59.000000Z',
'a2': '12.00', 'a3': 1, 'a4': 1754238678005782594}

param_types: {'a1': code: TIMESTAMP, 'a3': code: INT64, 'a4': code: INT64}
```

where 'a2' is the string representation that gets rendered.
I added code in do_execute_update to print out types on any exception and given
```python
sql: UPDATE i18n_company SET name = @a0, date_added = @a1, cents_paid = @a2, products_delivered = @a3 WHERE i18n_company.id = @a4
params: {'a0': 'Iñtërnâtiônàlizætiøn1', 'a1': '1799-01-31T23:59:59.000000Z', 'a2': '12.00', 'a3': 1, 'a4': 1046690653790600800}
param_types: {'a1': code: TIMESTAMP, 'a3': code: INT64, 'a4': code: INT64}
```
the printed out types are:
```shell
param: a0 value: Iñtërnâtiônàlizætiøn1 value_type: <class 'django.utils.safestring.SafeText'>
param: a1 value: 1799-01-31T23:59:59.000000Z value_type: <class 'spanner.dbapi.types.TimestampStr'>
param: a2 value: 12.00 value_type: <class 'str'>
param: a3 value: 1 value_type: <class 'int'>
param: a4 value: 1046690653790600800 value_type: <class 'int'>
```

and there we see that a2, the DecimalField was already given to use a string, as per the crash:
```shell
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 106, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Invalid value for bind parameter a2: Expected FLOAT64.
```

Resolving patch:
diff --git a/spanner/dbapi/parse_utils.py b/spanner/dbapi/parse_utils.py
--- a/spanner/dbapi/parse_utils.py
+++ b/spanner/dbapi/parse_utils.py
@@ -416,6 +416,8 @@ def infer_param_types(params, param_types):
     for key, value in params.items():
         if isinstance(value, bool):
             param_types = insert_key_in_param_types(key, param_types, spanner.param_types.BOOL)
+        elif isinstance(value, float):
+            param_types = insert_key_in_param_types(key, param_types, spanner.param_types.FLOAT64)
         elif isinstance(value, int):
             param_types = insert_key_in_param_types(key, param_types, spanner.param_types.INT64)
         elif isinstance(value, TimestampStr):
diff --git a/spanner/django/operations.py b/spanner/django/operations.py
--- a/spanner/django/operations.py
+++ b/spanner/django/operations.py
@@ -58,6 +58,15 @@ def adapt_datetimefield_value(self, value):
                 raise ValueError("Cloud Spanner does not support timezone-aware datetimes when USE_TZ is False.")
         return TimestampStr(value.isoformat(timespec='microseconds') + 'Z')
 
+    def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
+        """
+        Convert value from decimal.Decimal into float, for a direct mapping
+        and correct serialization with RPCs to Cloud Spanner.
+        """
+        if value is None:
+            return None
+        return float(value)
+
     def get_db_converters(self, expression):
         converters = super().get_db_converters(expression)
         internal_type = expression.output_field.get_internal_type()


Test patch:
diff --git a/tests/spanner/dbapi/test_parse_utils.py b/tests/spanner/dbapi/test_parse_utils.py
--- a/tests/spanner/dbapi/test_parse_utils.py
+++ b/tests/spanner/dbapi/test_parse_utils.py
@@ -319,9 +319,9 @@ def test_sql_pyformat_args_to_spanner_invalid(self):
     def test_infer_param_types(self):
         cases = [
             (
-                {'a1': 10, 'b1': '2005-08-30T01:01:01.000001Z', 'c1': '2019-12-05'},
+                {'a1': 10, 'b1': '2005-08-30T01:01:01.000001Z', 'c1': '2019-12-05', 'd1': 10.39},
                 None,
-                {'a1': spanner.param_types.INT64},
+                {'a1': spanner.param_types.INT64, 'd1': spanner.param_types.FLOAT64},
             ),
             (
                 {'a1': 10, 'b1': TimestampStr('2005-08-30T01:01:01.000001Z'), 'c1': '2019-12-05'},
@@ -345,12 +345,13 @@ def test_infer_param_types(self):
             ),
             ({'a1': 10, 'b1': 'aaaaa08-30T01:01:01.000001Z'}, None, {'a1': spanner.param_types.INT64}),
             (
-                {'a1': 10, 'b1': '2005-08-30T01:01:01.000001', 't1': True, 't2': False},
+                {'a1': 10, 'b1': '2005-08-30T01:01:01.000001', 't1': True, 't2': False, 'f1': 99e9},
                 None,
                 {
                     'a1': spanner.param_types.INT64,
                     't1': spanner.param_types.BOOL,
                     't2': spanner.param_types.BOOL,
+                    'f1': spanner.param_types.FLOAT64,
                 },
             ),
             (