Repository: googleapis/google-cloud-python

Issue:
New Standard SQL Data Types Not Supported in gcloud._helpers._rows_from_json() / _CELLDATA_FROM_JSON()
Standard SQL dialect support was added in 0.18.0, but current release does not support queries that return results that include data types that are not supported by Legacy SQL.

E.g. executing the following query using Standard SQL:

```
SELECT PARSE_DATE('%Y%m%d', '20160810') AS table_date;
```

results in 

```
rows = query.rows
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/gcloud/bigquery/query.py", line 204, in rows
    return _rows_from_json(self._properties.get('rows', ()), self.schema)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/gcloud/bigquery/_helpers.py", line 85, in _rows_
from_json
    converter = _CELLDATA_FROM_JSON[field.field_type]
KeyError: u'DATE'
```

Not a significant issue since the Legacy SQL Data types will work for my needs, but wanted to report it all the same.



Discussion:
@ToomToom Thanks for the report!  For future reference, here are the [docs for the column types returned when querying with the standard SQL dialect](https://cloud.google.com/bigquery/sql-reference/data-types).

Unfortunately there is no equivalent timezone functionality in the legacy SQL syntax, so this is a bit of an issue.

For anyone else blocked by this, the query actually works, and the raw data is available via

``` python
query._properties['rows']
```

If there is a preferable way to access this until fixed, then please recommend.

@tseaver isn't 'DATE' a valid type?
https://cloud.google.com/bigquery/sql-reference/data-types#date-type

It's listed here [https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/google/cloud/bigquery/_helpers.py#L81](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/5f1cda90d911f32f1f7f646a7711fdac63361447/google/cloud/bigquery/_helpers.py#L81).

@abovethewater, @ToomToom, @daspecster
- PR #2265 added support for the `DATE` type.
- We should be able to add the `INT64` and `FLOAT64` standard types as trivial aliases for the legacy `INT` and `FLOAT`.
- The standard `BYTES` type should be straightforward, if we can figure out how the back-end is expecting the values to be tunneled via JSON (I expect `base64`, but can't find it in the docs).
- Support for both `ARRAY` and `STRUCT` are going to require reworking the current implementation, as the field types are no longer constant strings.

- #2787 fixed the marshalling of `ARRAY` and `STRUCT` types, as well as adding the `BOOL` alias.
- #2803 fixed handling of `DATE`, `DATETIME`, `TIMESTAMP` fields.

AFAICT, the remaining standard SQL types which need support are `BYTES` (I just verified that the back-end marshals them as base64) and `TIME`.

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -14,6 +14,7 @@
 
 """Shared helper functions for BigQuery API classes."""
 
+import base64
 from collections import OrderedDict
 import datetime
 
@@ -22,6 +23,7 @@
 from google.cloud._helpers import _datetime_to_rfc3339
 from google.cloud._helpers import _microseconds_from_datetime
 from google.cloud._helpers import _RFC3339_NO_FRACTION
+from google.cloud._helpers import _time_from_iso8601_time_naive
 
 
 def _not_null(value, field):
@@ -47,6 +49,17 @@ def _bool_from_json(value, field):
         return value.lower() in ['t', 'true', '1']
 
 
+def _string_from_json(value, _):
+    """NOOP string -> string coercion"""
+    return value
+
+
+def _bytes_from_json(value, field):
+    """Base64-decode value"""
+    if _not_null(value, field):
+        return base64.decodestring(value)
+
+
 def _timestamp_from_json(value, field):
     """Coerce 'value' to a datetime, if set or not nullable."""
     if _not_null(value, field):
@@ -64,9 +77,17 @@ def _datetime_from_json(value, field):
 def _date_from_json(value, field):
     """Coerce 'value' to a datetime date, if set or not nullable"""
     if _not_null(value, field):
+        # value will be a string, in YYYY-MM-DD form.
         return _date_from_iso8601_date(value)
 
 
+def _time_from_json(value, field):
+    """Coerce 'value' to a datetime date, if set or not nullable"""
+    if _not_null(value, field):
+        # value will be a string, in HH:MM:SS form.
+        return _time_from_iso8601_time_naive(value)
+
+
 def _record_from_json(value, field):
     """Coerce 'value' to a mapping, if set or not nullable."""
     if _not_null(value, field):
@@ -82,11 +103,6 @@ def _record_from_json(value, field):
         return record
 
 
-def _string_from_json(value, _):
-    """NOOP string -> string coercion"""
-    return value
-
-
 _CELLDATA_FROM_JSON = {
     'INTEGER': _int_from_json,
     'INT64': _int_from_json,
@@ -94,11 +110,13 @@ def _string_from_json(value, _):
     'FLOAT64': _float_from_json,
     'BOOLEAN': _bool_from_json,
     'BOOL': _bool_from_json,
+    'STRING': _string_from_json,
+    'BYTES': _bytes_from_json,
     'TIMESTAMP': _timestamp_from_json,
     'DATETIME': _datetime_from_json,
     'DATE': _date_from_json,
+    'TIME': _time_from_json,
     'RECORD': _record_from_json,
-    'STRING': _string_from_json,
 }
 
 
@@ -121,6 +139,13 @@ def _bool_to_json(value):
     return value
 
 
+def _bytes_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, bytes):
+        value = base64.encodestring(value)
+    return value
+
+
 def _timestamp_to_json(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
@@ -142,6 +167,13 @@ def _date_to_json(value):
     return value
 
 
+def _time_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, datetime.time):
+        value = value.isoformat()
+    return value
+
+
 _SCALAR_VALUE_TO_JSON = {
     'INTEGER': _int_to_json,
     'INT64': _int_to_json,
@@ -149,9 +181,11 @@ def _date_to_json(value):
     'FLOAT64': _float_to_json,
     'BOOLEAN': _bool_to_json,
     'BOOL': _bool_to_json,
+    'BYTES': _bytes_to_json,
     'TIMESTAMP': _timestamp_to_json,
     'DATETIME': _datetime_to_json,
     'DATE': _date_to_json,
+    'TIME': _time_to_json,
 }
 
 
diff --git a/core/google/cloud/_helpers.py b/core/google/cloud/_helpers.py
--- a/core/google/cloud/_helpers.py
+++ b/core/google/cloud/_helpers.py
@@ -251,6 +251,19 @@ def _date_from_iso8601_date(value):
     return datetime.datetime.strptime(value, '%Y-%m-%d').date()
 
 
+def _time_from_iso8601_time_naive(value):
+    """Convert a zoneless ISO8601 time string to naive datetime time
+
+    :type value: str
+    :param value: The time string to convert
+
+    :rtype: :class:`datetime.time`
+    :returns: A datetime time object created from the string
+
+    """
+    return datetime.datetime.strptime(value, '%H:%M:%S').time()
+
+
 def _rfc3339_to_datetime(dt_str):
     """Convert a microsecond-precision timetamp to a native datetime.
 


Test patch:
diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py
--- a/bigquery/unit_tests/test__helpers.py
+++ b/bigquery/unit_tests/test__helpers.py
@@ -105,6 +105,44 @@ def test_w_value_other(self):
         self.assertFalse(coerced)
 
 
+class Test_string_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _string_from_json
+        return _string_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        self.assertIsNone(self._call_fut(None, _Field('REQUIRED')))
+
+    def test_w_string_value(self):
+        coerced = self._call_fut('Wonderful!', object())
+        self.assertEqual(coerced, 'Wonderful!')
+
+
+class Test_bytes_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _bytes_from_json
+        return _bytes_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field('REQUIRED'))
+
+    def test_w_base64_encoded_value(self):
+        import base64
+        expected = b'Wonderful!'
+        encoded = base64.encodestring(expected)
+        coerced = self._call_fut(encoded, object())
+        self.assertEqual(coerced, expected)
+
+
 class Test_timestamp_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -177,6 +215,27 @@ def test_w_string_value(self):
             datetime.date(1987, 9, 22))
 
 
+class Test_time_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _time_from_json
+        return _time_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field('REQUIRED'))
+
+    def test_w_string_value(self):
+        import datetime
+        coerced = self._call_fut('12:12:27', object())
+        self.assertEqual(
+            coerced,
+            datetime.time(12, 12, 27))
+
+
 class Test_record_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -238,23 +297,6 @@ def test_w_record_subfield(self):
         self.assertEqual(coerced, expected)
 
 
-class Test_string_from_json(unittest.TestCase):
-
-    def _call_fut(self, value, field):
-        from google.cloud.bigquery._helpers import _string_from_json
-        return _string_from_json(value, field)
-
-    def test_w_none_nullable(self):
-        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
-
-    def test_w_none_required(self):
-        self.assertIsNone(self._call_fut(None, _Field('RECORD')))
-
-    def test_w_string_value(self):
-        coerced = self._call_fut('Wonderful!', object())
-        self.assertEqual(coerced, 'Wonderful!')
-
-
 class Test_row_from_json(unittest.TestCase):
 
     def _call_fut(self, row, schema):
@@ -471,6 +513,23 @@ def test_w_string(self):
         self.assertEqual(self._call_fut('false'), 'false')
 
 
+class Test_bytes_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _bytes_to_json
+        return _bytes_to_json(value)
+
+    def test_w_non_bytes(self):
+        non_bytes = object()
+        self.assertIs(self._call_fut(non_bytes), non_bytes)
+
+    def test_w_bytes(self):
+        import base64
+        source = b'source'
+        expected = base64.encodestring(source)
+        self.assertEqual(self._call_fut(source), expected)
+
+
 class Test_timestamp_to_json(unittest.TestCase):
 
     def _call_fut(self, value):
@@ -522,6 +581,22 @@ def test_w_datetime(self):
         self.assertEqual(self._call_fut(when), '2016-12-03')
 
 
+class Test_time_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _time_to_json
+        return _time_to_json(value)
+
+    def test_w_string(self):
+        RFC3339 = '12:13:41'
+        self.assertEqual(self._call_fut(RFC3339), RFC3339)
+
+    def test_w_datetime(self):
+        import datetime
+        when = datetime.time(12, 13, 41)
+        self.assertEqual(self._call_fut(when), '12:13:41')
+
+
 class Test_ConfigurationProperty(unittest.TestCase):
 
     @staticmethod
diff --git a/core/unit_tests/test__helpers.py b/core/unit_tests/test__helpers.py
--- a/core/unit_tests/test__helpers.py
+++ b/core/unit_tests/test__helpers.py
@@ -265,6 +265,18 @@ def test_todays_date(self):
         self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY)
 
 
+class Test___time_from_iso8601_time_naive(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud._helpers import _time_from_iso8601_time_naive
+        return _time_from_iso8601_time_naive(value)
+
+    def test_todays_date(self):
+        import datetime
+        WHEN = datetime.time(12, 9, 42)
+        self.assertEqual(self._call_fut(("12:09:42")), WHEN)
+
+
 class Test__rfc3339_to_datetime(unittest.TestCase):
 
     def _call_fut(self, dt_str):
diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py
--- a/system_tests/bigquery.py
+++ b/system_tests/bigquery.py
@@ -479,12 +479,49 @@ def _job_done(instance):
         # raise an error, and that the job completed (in the `retry()`
         # above).
 
-    def test_sync_query_w_nested_arrays_and_structs(self):
+    def test_sync_query_w_standard_sql_types(self):
+        import datetime
+        from google.cloud._helpers import UTC
+        naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
+        stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat())
+        zoned = naive.replace(tzinfo=UTC)
         EXAMPLES = [
             {
                 'sql': 'SELECT 1',
                 'expected': 1,
             },
+            {
+                'sql': 'SELECT 1.3',
+                'expected': 1.3,
+            },
+            {
+                'sql': 'SELECT TRUE',
+                'expected': True,
+            },
+            {
+                'sql': 'SELECT "ABC"',
+                'expected': 'ABC',
+            },
+            {
+                'sql': 'SELECT CAST("foo" AS BYTES)',
+                'expected': b'foo',
+            },
+            {
+                'sql': 'SELECT TIMESTAMP "%s"' % (stamp,),
+                'expected': zoned,
+            },
+            {
+                'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive,
+            },
+            {
+                'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive.date(),
+            },
+            {
+                'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,),
+                'expected': naive.time(),
+            },
             {
                 'sql': 'SELECT (1, 2)',
                 'expected': {'_field_1': 1, '_field_2': 2},