Repository: googleapis/google-cloud-python

Issue:
BigQuery: '_time_from_json' raises for values w/ sub-second precision
- OS: Arch Linux
- relevant conda versions:
```
# packages in environment at /home/phillip/miniconda3/envs/ibis36:
#
# Name                    Version                   Build  Channel
google-cloud-bigquery     1.4.0                      py_0    conda-forge
google-cloud-core         0.28.1                     py_0    conda-forge
```

- Steps to reproduce:

```python
import google.cloud.bigquery as bq
client = bq.Client('ibis-gbq')
sql = """
SELECT TIME(`creation_date`) AS `created_time`
FROM `bigquery-public-data.stackoverflow.posts_questions`
"""
query = client.query(sql)
df = query.result().to_dataframe()
```

- Traceback:

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-27-caeea53c88e2> in <module>()
----> 1 df = query.result().to_dataframe()

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/bigquery/table.py in to_dataframe(self)
   1145
   1146         column_headers = [field.name for field in self.schema]
-> 1147         rows = [row.values() for row in iter(self)]
   1148
   1149         return pandas.DataFrame(rows, columns=column_headers)

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/bigquery/table.py in <listcomp>(.0)
   1145
   1146         column_headers = [field.name for field in self.schema]
-> 1147         rows = [row.values() for row in iter(self)]
   1148
   1149         return pandas.DataFrame(rows, columns=column_headers)

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/api_core/page_iterator.py in _items_iter(self)
    198         """Iterator for each item returned."""
    199         for page in self._page_iter(increment=False):
--> 200             for item in page:
    201                 self.num_results += 1
    202                 yield item

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/api_core/page_iterator.py in next(self)
    123         """Get the next value in the page."""
    124         item = six.next(self._item_iter)
--> 125         result = self._item_to_value(self._parent, item)
    126         # Since we've successfully got the next value from the
    127         # iterator, we update the number of remaining.

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/bigquery/table.py in _item_to_row(iterator, resource)
   1314     :returns: The next row in the page.
   1315     """
-> 1316     return Row(_helpers._row_tuple_from_json(resource, iterator.schema),
   1317                iterator._field_to_index)
   1318

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/bigquery/_helpers.py in _row_tuple_from_json(row, schema)
    210                              for item in cell['v']])
    211         else:
--> 212             row_data.append(converter(cell['v'], field))
    213
    214     return tuple(row_data)

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/bigquery/_helpers.py in _time_from_json(value, field)
    144     if _not_null(value, field):
    145         # value will be a string, in HH:MM:SS form.
--> 146         return _time_from_iso8601_time_naive(value)
    147
    148

~/miniconda3/envs/ibis36/lib/python3.6/site-packages/google/cloud/_helpers.py in _time_from_iso8601_time_naive(value)
    259
    260     """
--> 261     return datetime.datetime.strptime(value, '%H:%M:%S').time()
    262
    263

~/miniconda3/envs/ibis36/lib/python3.6/_strptime.py in _strptime_datetime(cls, data_string, format)
    563     """Return a class cls instance based on the input string and the
    564     format string."""
--> 565     tt, fraction = _strptime(data_string, format)
    566     tzname, gmtoff = tt[-2:]
    567     args = tt[:6] + (fraction,)

~/miniconda3/envs/ibis36/lib/python3.6/_strptime.py in _strptime(data_string, format)
    363     if len(data_string) != found.end():
    364         raise ValueError("unconverted data remains: %s" %
--> 365                           data_string[found.end():])
    366
    367     iso_year = year = None

ValueError: unconverted data remains: .590000
```


Discussion:
I can reproduce locally, with the correction that the field name in the query should be `creation_date`, rather than `created_date`.
@tseaver Whoops, thanks. Fixed.
@cpcloud Thanks very much for the report.  #5753 will address the issue for BigQuery.  I will leave this issue open to propagate the fix to 'google.cloud._helpers._time_from_iso8601_time_naive'.

Resolving patch:
diff --git a/core/google/cloud/_helpers.py b/core/google/cloud/_helpers.py
--- a/core/google/cloud/_helpers.py
+++ b/core/google/cloud/_helpers.py
@@ -43,6 +43,8 @@
 _NOW = datetime.datetime.utcnow  # To be replaced by tests.
 _RFC3339_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ'
 _RFC3339_NO_FRACTION = '%Y-%m-%dT%H:%M:%S'
+_TIMEONLY_W_MICROS = '%H:%M:%S.%f'
+_TIMEONLY_NO_FRACTION = '%H:%M:%S'
 # datetime.strptime cannot handle nanosecond precision:  parse w/ regex
 _RFC3339_NANOS = re.compile(r"""
     (?P<no_fraction>
@@ -256,9 +258,15 @@ def _time_from_iso8601_time_naive(value):
 
     :rtype: :class:`datetime.time`
     :returns: A datetime time object created from the string
-
+    :raises ValueError: if the value does not match a known format.
     """
-    return datetime.datetime.strptime(value, '%H:%M:%S').time()
+    if len(value) == 8:  # HH:MM:SS
+        fmt = _TIMEONLY_NO_FRACTION
+    elif len(value) == 15:  # HH:MM:SS.micros
+        fmt = _TIMEONLY_W_MICROS
+    else:
+        raise ValueError("Unknown time format: {}".format(value))
+    return datetime.datetime.strptime(value, fmt).time()
 
 
 def _rfc3339_to_datetime(dt_str):


Test patch:
diff --git a/core/tests/unit/test__helpers.py b/core/tests/unit/test__helpers.py
--- a/core/tests/unit/test__helpers.py
+++ b/core/tests/unit/test__helpers.py
@@ -298,6 +298,16 @@ def test_todays_date(self):
         WHEN = datetime.time(12, 9, 42)
         self.assertEqual(self._call_fut(("12:09:42")), WHEN)
 
+    def test_w_microseconds(self):
+        import datetime
+
+        WHEN = datetime.time(12, 9, 42, 123456)
+        self.assertEqual(self._call_fut(("12:09:42.123456")), WHEN)
+
+    def test_w_millis_fail(self):
+        with self.assertRaises(ValueError):
+            self._call_fut("12:09:42.123")
+
 
 class Test__rfc3339_to_datetime(unittest.TestCase):