Repository: googleapis/python-spanner-django

Issue:
django: store TimeField as Timestamp
Probably `TimeField` should be stored as a `Timestamp` in Spanner rather than as STRING, since relevant database functions like EXTRACT don't work on STRING.

Example failure:
```
======================================================================
ERROR: test_hour_lookups (lookup.test_timefield.TimeFieldLookupTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 79, in next
    return six.next(self._wrapped)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 392, in __next__
    return self._next()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/grpc/_channel.py", line 561, in _next
    raise self
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "No matching signature for function EXTRACT for argument types: DATE_TIME_PART FROM STRING. Supported signatures: EXTRACT(DATE_TIME_PART FROM DATE); EXTRACT(DATE_TIME_PART FROM TIMESTAMP [AT TIME ZONE STRING]) [at 1:88]\n...desc`, lookup_alarm.time FROM lookup_alarm WHERE EXTRACT(hour FROM lookup_...\n                                                    ^"
	debug_error_string = "{"created":"@1576094453.117519159","description":"Error received from peer ipv4:172.217.10.138:443","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"No matching signature for function EXTRACT for argument types: DATE_TIME_PART FROM STRING. Supported signatures: EXTRACT(DATE_TIME_PART FROM DATE); EXTRACT(DATE_TIME_PART FROM TIMESTAMP [AT TIME ZONE STRING]) [at 1:88]\n...desc`, lookup_alarm.time FROM lookup_alarm WHERE EXTRACT(hour FROM lookup_...\n                                                    ^","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/tim/code/django/tests/lookup/test_timefield.py", line 19, in test_hour_lookups
    ordered=False
  File "/home/tim/code/django/django/test/testcases.py", line 1047, in assertQuerysetEqual
    items = map(transform, qs)
  File "/home/tim/code/django/django/db/models/query.py", line 274, in __iter__
    self._fetch_all()
  File "/home/tim/code/django/django/db/models/query.py", line 1242, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
  File "/home/tim/code/django/django/db/models/query.py", line 55, in __iter__
    results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1166, in execute_sql
    return list(result)
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1545, in cursor_iter
    for rows in iter((lambda: cursor.fetchmany(itersize)), sentinel):
  File "/home/tim/code/django/django/db/models/sql/compiler.py", line 1545, in <lambda>
    for rows in iter((lambda: cursor.fetchmany(itersize)), sentinel):
  File "/home/tim/code/django/django/db/utils.py", line 96, in inner
    return func(*args, **kwargs)
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 247, in fetchmany
    items.append(tuple(self.__next__()))
  File "/home/tim/code/spanner-orm/spanner/dbapi/cursor.py", line 210, in __next__
    return next(self.__itr)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 143, in __iter__
    self._consume_next()
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 116, in _consume_next
    response = six.next(self._response_iterator)
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/cloud/spanner_v1/snapshot.py", line 45, in _restart_on_unavailable
    for item in iterator:
  File "/home/tim/.virtualenvs/django37/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 81, in next
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 No matching signature for function EXTRACT for argument types: DATE_TIME_PART FROM STRING. Supported signatures: EXTRACT(DATE_TIME_PART FROM DATE); EXTRACT(DATE_TIME_PART FROM TIMESTAMP [AT TIME ZONE STRING]) [at 1:88]\n...desc`, lookup_alarm.time FROM lookup_alarm WHERE EXTRACT(hour FROM lookup_...\n                                                    ^
```


Discussion:

Resolving patch:
diff --git a/spanner/django/base.py b/spanner/django/base.py
--- a/spanner/django/base.py
+++ b/spanner/django/base.py
@@ -56,8 +56,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'SmallAutoField': 'INT64',
         'SmallIntegerField': 'INT64',
         'TextField': 'STRING(MAX)',
-        # With or without the time zone, Spanner expects the time field as a string.
-        'TimeField': 'STRING(50)',
+        'TimeField': 'TIMESTAMP',
         # A UUID4 like 'd9c388b1-184d-4511-a818-3d598cc2f847', 16 bytes, with 4 dashes.
         'UUIDField': 'STRING(36)',
     }
diff --git a/spanner/django/features.py b/spanner/django/features.py
--- a/spanner/django/features.py
+++ b/spanner/django/features.py
@@ -54,12 +54,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         # https://github.com/orijtech/spanner-orm/issues/170
         'model_fields.test_datetimefield.DateTimeFieldTests.test_lookup_date_with_use_tz',
         'model_fields.test_datetimefield.DateTimeFieldTests.test_lookup_date_without_use_tz',
-        # Store TimeField as Timestamp:
-        # https://github.com/orijtech/spanner-orm/issues/167
-        'lookup.test_timefield.TimeFieldLookupTests.test_hour_lookups',
-        'lookup.test_timefield.TimeFieldLookupTests.test_minute_lookups',
-        'lookup.test_timefield.TimeFieldLookupTests.test_second_lookups',
-        'model_fields.test_datetimefield.DateTimeFieldTests.test_datetimes_save_completely',
         # DecimalField lookups crash:
         # https://github.com/orijtech/spanner-orm/issues/168
         'lookup.test_decimalfield.DecimalFieldLookupTests.test_gt',
diff --git a/spanner/django/operations.py b/spanner/django/operations.py
--- a/spanner/django/operations.py
+++ b/spanner/django/operations.py
@@ -1,6 +1,6 @@
 import re
 from base64 import b64decode
-from datetime import datetime
+from datetime import datetime, time
 from decimal import Decimal
 from uuid import UUID
 
@@ -74,6 +74,12 @@ def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
             return None
         return float(value)
 
+    def adapt_timefield_value(self, value):
+        if value is None:
+            return None
+        # Column is TIMESTAMP, so prefix a dummy date to the datetime.time.
+        return TimestampStr('0001-01-01T' + value.isoformat(timespec='microseconds') + 'Z')
+
     def get_db_converters(self, expression):
         converters = super().get_db_converters(expression)
         internal_type = expression.output_field.get_internal_type()
@@ -81,6 +87,8 @@ def get_db_converters(self, expression):
             converters.append(self.convert_datetimefield_value)
         elif internal_type == 'DecimalField':
             converters.append(self.convert_decimalfield_value)
+        elif internal_type == 'TimeField':
+            converters.append(self.convert_timefield_value)
         elif internal_type == 'BinaryField':
             converters.append(self.convert_binaryfield_value)
         elif internal_type == 'UUIDField':
@@ -113,6 +121,12 @@ def convert_decimalfield_value(self, value, expression, connection):
         # Cloud Spanner returns a float.
         return Decimal(str(value))
 
+    def convert_timefield_value(self, value, expression, connection):
+        if value is None:
+            return value
+        # Convert DatetimeWithNanoseconds to time.
+        return time(value.hour, value.minute, value.second, value.microsecond)
+
     def convert_uuidfield_value(self, value, expression, connection):
         if value is not None:
             value = UUID(value)
@@ -127,6 +141,10 @@ def datetime_extract_sql(self, lookup_type, field_name, tzname):
         lookup_type = self.extract_names.get(lookup_type, lookup_type)
         return 'EXTRACT(%s FROM %s AT TIME ZONE "%s")' % (lookup_type, field_name, tzname)
 
+    def time_extract_sql(self, lookup_type, field_name):
+        # Time is stored as TIMESTAMP with UTC time zone.
+        return 'EXTRACT(%s FROM %s AT TIME ZONE "UTC")' % (lookup_type, field_name)
+
     def date_trunc_sql(self, lookup_type, field_name):
         # https://cloud.google.com/spanner/docs/functions-and-operators#date_trunc
         return 'DATE_TRUNC(%s, %s)' % (field_name, lookup_type)


Test patch: