Repository: googleapis/python-bigquery

Issue:
Timestamp loss of precision in insert_rows
Certain values of `datetime` lose a single microsecond on their way into BigQuery. This appears to happen because of the JSON rendering of timestamps as a floating point number of seconds.

#### Environment details

  - OS type and version: Ubuntu Bionic
  - Python version: 3.6.9
  - pip version: 9.0.1
  - `google-cloud-bigquery` version: 1.28.0

#### Steps to reproduce

Here is an example of a `datetime` value that loses a microsecond:
```py
datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=datetime.timezone(datetime.timedelta(0), 'UTC'))
```

As a full integration repro, you could try something like this:

  1. Create a BigQuery table with a `TIMESTAMP` column.
  2. Use `insert_rows` to insert the value above.
  3. Inspect the results in the Console, where it will show as `2020-11-17 01:06:52.353794 UTC` instead of `2020-11-17 01:06:52.353795 UTC`.

#### Code example

An easier repro hints at the root cause. By looking at the output of `_timestamp_to_json_row`, it appears that the problematic values are rounded down to microsecond precision.

```python
import datetime
from google.cloud import bigquery as bq
ts = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=datetime.timezone(datetime.timedelta(0), 'UTC'))
print(bq._helpers._timestamp_to_json_row(ts))
```
This will print:
```
1605575212.3537948
```

Perhaps the solution is to add a half microsecond (0.5e-6) so that truncating results in rounding to the nearest microsecond.



Discussion:

Resolving patch:
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
--- a/google/cloud/bigquery/_helpers.py
+++ b/google/cloud/bigquery/_helpers.py
@@ -23,7 +23,7 @@
 from google.cloud._helpers import UTC
 from google.cloud._helpers import _date_from_iso8601_date
 from google.cloud._helpers import _datetime_from_microseconds
-from google.cloud._helpers import _microseconds_from_datetime
+from google.cloud._helpers import _RFC3339_MICROS
 from google.cloud._helpers import _RFC3339_NO_FRACTION
 from google.cloud._helpers import _to_bytes
 
@@ -313,12 +313,9 @@ def _timestamp_to_json_parameter(value):
 
 
 def _timestamp_to_json_row(value):
-    """Coerce 'value' to an JSON-compatible representation.
-
-    This version returns floating-point seconds value used in row data.
-    """
+    """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
-        value = _microseconds_from_datetime(value) * 1e-6
+        value = value.strftime(_RFC3339_MICROS)
     return value
 
 


Test patch:
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
--- a/tests/unit/test__helpers.py
+++ b/tests/unit/test__helpers.py
@@ -728,10 +728,14 @@ def test_w_string(self):
         self.assertEqual(self._call_fut(ZULU), ZULU)
 
     def test_w_datetime(self):
-        from google.cloud._helpers import _microseconds_from_datetime
-
         when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
-        self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6)
+        self.assertEqual(self._call_fut(when), "2016-12-20T15:58:27.339328Z")
+
+    def test_w_datetime_w_utc_zone(self):
+        from google.cloud._helpers import UTC
+
+        when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC)
+        self.assertEqual(self._call_fut(when), "2020-11-17T01:06:52.353795Z")
 
 
 class Test_datetime_to_json(unittest.TestCase):
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -5804,7 +5804,7 @@ def test_insert_rows_w_schema(self):
         import datetime
         from google.cloud._helpers import UTC
         from google.cloud._helpers import _datetime_to_rfc3339
-        from google.cloud._helpers import _microseconds_from_datetime
+        from google.cloud._helpers import _RFC3339_MICROS
         from google.cloud.bigquery.schema import SchemaField
 
         WHEN_TS = 1437767599.006
@@ -5834,7 +5834,7 @@ def _row_data(row):
             result = {"full_name": row[0], "age": str(row[1])}
             joined = row[2]
             if isinstance(joined, datetime.datetime):
-                joined = _microseconds_from_datetime(joined) * 1e-6
+                joined = joined.strftime(_RFC3339_MICROS)
             if joined is not None:
                 result["joined"] = joined
             return result
@@ -5864,7 +5864,7 @@ def test_insert_rows_w_list_of_dictionaries(self):
         import datetime
         from google.cloud._helpers import UTC
         from google.cloud._helpers import _datetime_to_rfc3339
-        from google.cloud._helpers import _microseconds_from_datetime
+        from google.cloud._helpers import _RFC3339_MICROS
         from google.cloud.bigquery.schema import SchemaField
         from google.cloud.bigquery.table import Table
 
@@ -5910,7 +5910,7 @@ def _row_data(row):
                 row = copy.deepcopy(row)
                 del row["joined"]
             elif isinstance(joined, datetime.datetime):
-                row["joined"] = _microseconds_from_datetime(joined) * 1e-6
+                row["joined"] = joined.strftime(_RFC3339_MICROS)
             row["age"] = str(row["age"])
             return row
 
@@ -6109,16 +6109,16 @@ def test_insert_rows_w_repeated_fields(self):
                             {
                                 "score": "12",
                                 "times": [
-                                    1543665600.0,  # 2018-12-01 12:00 UTC
-                                    1543669200.0,  # 2018-12-01 13:00 UTC
+                                    "2018-12-01T12:00:00.000000Z",
+                                    "2018-12-01T13:00:00.000000Z",
                                 ],
                                 "distances": [1.25, 2.5],
                             },
                             {
                                 "score": "13",
                                 "times": [
-                                    1543752000.0,  # 2018-12-02 12:00 UTC
-                                    1543755600.0,  # 2018-12-02 13:00 UTC
+                                    "2018-12-02T12:00:00.000000Z",
+                                    "2018-12-02T13:00:00.000000Z",
                                 ],
                                 "distances": [-1.25, -2.5],
                             },