Repository: googleapis/python-bigquery

Issue:
BigQuery python cli fails to change time, BUT does changes timezone for TIMESTAMP which leads to an incorrect time
This code works with with the python package
   **google-cloud-bigquery==2.2.0** 
but it does not work with this package
   **google-cloud-bigquery==2.6.0**

What you expected to happen:

I expect the client library to take a datetime of
 '_2017-11-01 04:08 AM -07:00_'    and convert it to
 '_2017-11-01 11:08 AM -00:00_' in big query. but instead it uploads it to
 '_2017-11-01 04:08 AM -00:00_'

Steps to reproduce:
```
import datetime
from dateutil.tz import tzoffset
from google.cloud.bigquery import SchemaField, Table

schema = [SchemaField('c', 'TIMESTAMP', 'NULLABLE', None, (), None)] 
problem_date = datetime.datetime(2017, 11, 1, 4, 8, tzinfo=tzoffset(None, -25200))
table_name = ‘my_test_table’
dataset = <an existing dataset>

bigquery = bigquery.Client(credentials=<somecreds>, project=<ourproj>)

dataset_ref = bigquery.dataset(dataset, project=<ourproj>)
table_ref = dataset_ref.table(table_name)
table = bigquery.create_table(Table(table_ref, schema=schema))
errors = bigquery.insert_rows(table, [[problem_date]])
```

#### Other info
-------------------------------
I worked with the google BigQuery team and here is their [feedback](https://issuetracker.google.com/issues/175849093).
Hello,

I have been investigating the issue you described and it seems like this change of behaviour of the python client library was introduced in the version 2.5.0 [1] with the pull request #393 [2], whose objective was to fix a previous issue with floating points and timestamps in the method `insert_rows`.

I have been able to reproduce your issue in new versions after 2.5.0 while it does not happen in the previous ones. The suggested way to proceed here would be to open an issue in the Github repository [3] so you can transmit your findings to the Engineering Team, since the code is open source .

Alternatively, you might want to add the following line of code, so the timezone is converted to UTC before trying to insert the row into BigQuery. Please let me know if this workaround worked for you:
```
problem_date = datetime.datetime(2017, 11, 1, 4, 8, tzinfo=tzoffset(None, -25200))
problem_date = problem_date.astimezone(datetime.timezone.utc)
```

[1]:
https://github.com/googleapis/python-bigquery/releases/tag/v2.5.0
[2]:
https://github.com/googleapis/python-bigquery/pull/393
[3]:
https://github.com/googleapis/python-bigquery/issues

-------------------------------

We will look into the feasibility of the workaround





Discussion:
I was able to reproduce the issue with the latest BigQuery version, but also with version `2.2.0`, which doesn't match the reported behavior (maybe the error was on my side).

In any case, `tzinfo` does not seem to be respected and we need to fix it.

Resolving patch:
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
--- a/google/cloud/bigquery/_helpers.py
+++ b/google/cloud/bigquery/_helpers.py
@@ -315,6 +315,10 @@ def _timestamp_to_json_parameter(value):
 def _timestamp_to_json_row(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
+        # For naive datetime objects UTC timezone is assumed, thus we format
+        # those to string directly without conversion.
+        if value.tzinfo is not None:
+            value = value.astimezone(UTC)
         value = value.strftime(_RFC3339_MICROS)
     return value
 
@@ -322,6 +326,10 @@ def _timestamp_to_json_row(value):
 def _datetime_to_json(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
+        # For naive datetime objects UTC timezone is assumed, thus we format
+        # those to string directly without conversion.
+        if value.tzinfo is not None:
+            value = value.astimezone(UTC)
         value = value.strftime(_RFC3339_MICROS_NO_ZULU)
     return value
 


Test patch:
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
--- a/tests/unit/test__helpers.py
+++ b/tests/unit/test__helpers.py
@@ -420,13 +420,13 @@ def _call_fut(self, row, schema):
     def test_w_single_scalar_column(self):
         # SELECT 1 AS col
         col = _Field("REQUIRED", "col", "INTEGER")
-        row = {u"f": [{u"v": u"1"}]}
+        row = {"f": [{"v": "1"}]}
         self.assertEqual(self._call_fut(row, schema=[col]), (1,))
 
     def test_w_single_scalar_geography_column(self):
         # SELECT 1 AS col
         col = _Field("REQUIRED", "geo", "GEOGRAPHY")
-        row = {u"f": [{u"v": u"POINT(1, 2)"}]}
+        row = {"f": [{"v": "POINT(1, 2)"}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ("POINT(1, 2)",))
 
     def test_w_single_struct_column(self):
@@ -434,13 +434,13 @@ def test_w_single_struct_column(self):
         sub_1 = _Field("REQUIRED", "sub_1", "INTEGER")
         sub_2 = _Field("REQUIRED", "sub_2", "INTEGER")
         col = _Field("REQUIRED", "col", "RECORD", fields=[sub_1, sub_2])
-        row = {u"f": [{u"v": {u"f": [{u"v": u"1"}, {u"v": u"2"}]}}]}
+        row = {"f": [{"v": {"f": [{"v": "1"}, {"v": "2"}]}}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ({"sub_1": 1, "sub_2": 2},))
 
     def test_w_single_array_column(self):
         # SELECT [1, 2, 3] as col
         col = _Field("REPEATED", "col", "INTEGER")
-        row = {u"f": [{u"v": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]}]}
+        row = {"f": [{"v": [{"v": "1"}, {"v": "2"}, {"v": "3"}]}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ([1, 2, 3],))
 
     def test_w_struct_w_nested_array_column(self):
@@ -450,13 +450,13 @@ def test_w_struct_w_nested_array_column(self):
         third = _Field("REPEATED", "third", "INTEGER")
         col = _Field("REQUIRED", "col", "RECORD", fields=[first, second, third])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": {
-                        u"f": [
-                            {u"v": [{u"v": u"1"}, {u"v": u"2"}]},
-                            {u"v": u"3"},
-                            {u"v": [{u"v": u"4"}, {u"v": u"5"}]},
+                    "v": {
+                        "f": [
+                            {"v": [{"v": "1"}, {"v": "2"}]},
+                            {"v": "3"},
+                            {"v": [{"v": "4"}, {"v": "5"}]},
                         ]
                     }
                 }
@@ -464,7 +464,7 @@ def test_w_struct_w_nested_array_column(self):
         }
         self.assertEqual(
             self._call_fut(row, schema=[col]),
-            ({u"first": [1, 2], u"second": 3, u"third": [4, 5]},),
+            ({"first": [1, 2], "second": 3, "third": [4, 5]},),
         )
 
     def test_w_array_of_struct(self):
@@ -474,11 +474,11 @@ def test_w_array_of_struct(self):
         third = _Field("REQUIRED", "third", "INTEGER")
         col = _Field("REPEATED", "col", "RECORD", fields=[first, second, third])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": [
-                        {u"v": {u"f": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]}},
-                        {u"v": {u"f": [{u"v": u"4"}, {u"v": u"5"}, {u"v": u"6"}]}},
+                    "v": [
+                        {"v": {"f": [{"v": "1"}, {"v": "2"}, {"v": "3"}]}},
+                        {"v": {"f": [{"v": "4"}, {"v": "5"}, {"v": "6"}]}},
                     ]
                 }
             ]
@@ -487,8 +487,8 @@ def test_w_array_of_struct(self):
             self._call_fut(row, schema=[col]),
             (
                 [
-                    {u"first": 1, u"second": 2, u"third": 3},
-                    {u"first": 4, u"second": 5, u"third": 6},
+                    {"first": 1, "second": 2, "third": 3},
+                    {"first": 4, "second": 5, "third": 6},
                 ],
             ),
         )
@@ -499,32 +499,25 @@ def test_w_array_of_struct_w_array(self):
         second = _Field("REQUIRED", "second", "INTEGER")
         col = _Field("REPEATED", "col", "RECORD", fields=[first, second])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": [
-                        {
-                            u"v": {
-                                u"f": [
-                                    {u"v": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]},
-                                    {u"v": u"4"},
-                                ]
-                            }
-                        },
+                    "v": [
                         {
-                            u"v": {
-                                u"f": [
-                                    {u"v": [{u"v": u"5"}, {u"v": u"6"}]},
-                                    {u"v": u"7"},
+                            "v": {
+                                "f": [
+                                    {"v": [{"v": "1"}, {"v": "2"}, {"v": "3"}]},
+                                    {"v": "4"},
                                 ]
                             }
                         },
+                        {"v": {"f": [{"v": [{"v": "5"}, {"v": "6"}]}, {"v": "7"}]}},
                     ]
                 }
             ]
         }
         self.assertEqual(
             self._call_fut(row, schema=[col]),
-            ([{u"first": [1, 2, 3], u"second": 4}, {u"first": [5, 6], u"second": 7}],),
+            ([{"first": [1, 2, 3], "second": 4}, {"first": [5, 6], "second": 7}],),
         )
 
 
@@ -673,7 +666,7 @@ def test_w_non_bytes(self):
 
     def test_w_bytes(self):
         source = b"source"
-        expected = u"c291cmNl"
+        expected = "c291cmNl"
         converted = self._call_fut(source)
         self.assertEqual(converted, expected)
 
@@ -726,7 +719,7 @@ def test_w_string(self):
         ZULU = "2016-12-20 15:58:27.339328+00:00"
         self.assertEqual(self._call_fut(ZULU), ZULU)
 
-    def test_w_datetime(self):
+    def test_w_datetime_no_zone(self):
         when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
         self.assertEqual(self._call_fut(when), "2016-12-20T15:58:27.339328Z")
 
@@ -736,6 +729,14 @@ def test_w_datetime_w_utc_zone(self):
         when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC)
         self.assertEqual(self._call_fut(when), "2020-11-17T01:06:52.353795Z")
 
+    def test_w_datetime_w_non_utc_zone(self):
+        class EstZone(datetime.tzinfo):
+            def utcoffset(self, _):
+                return datetime.timedelta(minutes=-300)
+
+        when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=EstZone())
+        self.assertEqual(self._call_fut(when), "2020-11-17T06:06:52.353795Z")
+
 
 class Test_datetime_to_json(unittest.TestCase):
     def _call_fut(self, value):
@@ -753,6 +754,14 @@ def test_w_datetime(self):
         when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=UTC)
         self.assertEqual(self._call_fut(when), "2016-12-03T14:11:27.123456")
 
+    def test_w_datetime_w_non_utc_zone(self):
+        class EstZone(datetime.tzinfo):
+            def utcoffset(self, _):
+                return datetime.timedelta(minutes=-300)
+
+        when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=EstZone())
+        self.assertEqual(self._call_fut(when), "2016-12-03T19:11:27.123456")
+
 
 class Test_date_to_json(unittest.TestCase):
     def _call_fut(self, value):