Repository: googleapis/google-cloud-python

Issue:
Query paramters need to coerce native Python values
If the user passes in non-string values (e.g., for `INT64`, which must be tunnelled via strings because JS reasons, or `TIMESTAMP`, `DATETIME`, `DATE`), the parameter must coerce the value appropriately inside its `to_api_repr()` method.

h/t to @dwsupplee for the prod.


Discussion:

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -15,9 +15,13 @@
 """Shared helper functions for BigQuery API classes."""
 
 from collections import OrderedDict
+import datetime
 
-from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _date_from_iso8601_date
+from google.cloud._helpers import _datetime_from_microseconds
+from google.cloud._helpers import _datetime_to_rfc3339
+from google.cloud._helpers import _microseconds_from_datetime
+from google.cloud._helpers import _RFC3339_NO_FRACTION
 
 
 def _not_null(value, field):
@@ -43,13 +47,20 @@ def _bool_from_json(value, field):
         return value.lower() in ['t', 'true', '1']
 
 
-def _datetime_from_json(value, field):
+def _timestamp_from_json(value, field):
     """Coerce 'value' to a datetime, if set or not nullable."""
     if _not_null(value, field):
         # value will be a float in seconds, to microsecond precision, in UTC.
         return _datetime_from_microseconds(1e6 * float(value))
 
 
+def _datetime_from_json(value, field):
+    """Coerce 'value' to a datetime, if set or not nullable."""
+    if _not_null(value, field):
+        # value will be a string, in YYYY-MM-DDTHH:MM:SS form.
+        return datetime.datetime.strptime(value, _RFC3339_NO_FRACTION)
+
+
 def _date_from_json(value, field):
     """Coerce 'value' to a datetime date, if set or not nullable"""
     if _not_null(value, field):
@@ -83,13 +94,67 @@ def _string_from_json(value, _):
     'FLOAT64': _float_from_json,
     'BOOLEAN': _bool_from_json,
     'BOOL': _bool_from_json,
-    'TIMESTAMP': _datetime_from_json,
+    'TIMESTAMP': _timestamp_from_json,
+    'DATETIME': _datetime_from_json,
     'DATE': _date_from_json,
     'RECORD': _record_from_json,
     'STRING': _string_from_json,
 }
 
 
+def _int_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, int):
+        value = str(value)
+    return value
+
+
+def _float_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    return value
+
+
+def _bool_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, bool):
+        value = 'true' if value else 'false'
+    return value
+
+
+def _timestamp_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, datetime.datetime):
+        value = _microseconds_from_datetime(value) / 1.0e6
+    return value
+
+
+def _datetime_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, datetime.datetime):
+        value = _datetime_to_rfc3339(value)
+    return value
+
+
+def _date_to_json(value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, datetime.date):
+        value = value.isoformat()
+    return value
+
+
+_SCALAR_VALUE_TO_JSON = {
+    'INTEGER': _int_to_json,
+    'INT64': _int_to_json,
+    'FLOAT': _float_to_json,
+    'FLOAT64': _float_to_json,
+    'BOOLEAN': _bool_to_json,
+    'BOOL': _bool_to_json,
+    'TIMESTAMP': _timestamp_to_json,
+    'DATETIME': _datetime_to_json,
+    'DATE': _date_to_json,
+}
+
+
 def _row_from_json(row, schema):
     """Convert JSON row data to row with appropriate types.
 
@@ -262,8 +327,8 @@ class ScalarQueryParameter(AbstractQueryParameter):
                  paramter can only be addressed via position (``?``).
 
     :type type_: str
-    :param type_: name of parameter type.  One of `'STRING'`, `'INT64'`,
-                  `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
+    :param type_: name of parameter type.  One of 'STRING', 'INT64',
+                  'FLOAT64', 'BOOL', 'TIMESTAMP', 'DATETIME', or 'DATE'.
 
     :type value: str, int, float, bool, :class:`datetime.datetime`, or
                  :class:`datetime.date`.
@@ -279,8 +344,9 @@ def positional(cls, type_, value):
         """Factory for positional paramters.
 
         :type type_: str
-        :param type_: name of paramter type.  One of `'STRING'`, `'INT64'`,
-                      `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
+        :param type_:
+            name of paramter type.  One of 'STRING', 'INT64',
+            'FLOAT64', 'BOOL', 'TIMESTAMP', 'DATETIME', or 'DATE'.
 
         :type value: str, int, float, bool, :class:`datetime.datetime`, or
                      :class:`datetime.date`.
@@ -313,12 +379,16 @@ def to_api_repr(self):
         :rtype: dict
         :returns: JSON mapping
         """
+        value = self.value
+        converter = _SCALAR_VALUE_TO_JSON.get(self.type_)
+        if converter is not None:
+            value = converter(value)
         resource = {
             'parameterType': {
                 'type': self.type_,
             },
             'parameterValue': {
-                'value': self.value,
+                'value': value,
             },
         }
         if self.name is not None:
@@ -386,12 +456,16 @@ def to_api_repr(self):
         :rtype: dict
         :returns: JSON mapping
         """
+        values = self.values
+        converter = _SCALAR_VALUE_TO_JSON.get(self.array_type)
+        if converter is not None:
+            values = [converter(value) for value in values]
         resource = {
             'parameterType': {
                 'arrayType': self.array_type,
             },
             'parameterValue': {
-                'arrayValues': self.values,
+                'arrayValues': values,
             },
         }
         if self.name is not None:
@@ -458,12 +532,19 @@ def to_api_repr(self):
             {'name': key, 'type': value}
             for key, value in self.struct_types.items()
         ]
+        values = {}
+        for name, value in self.struct_values.items():
+            converter = _SCALAR_VALUE_TO_JSON.get(self.struct_types[name])
+            if converter is not None:
+                value = converter(value)
+            values[name] = value
+
         resource = {
             'parameterType': {
                 'structTypes': types,
             },
             'parameterValue': {
-                'structValues': self.struct_values,
+                'structValues': values,
             },
         }
         if self.name is not None:


Test patch:
diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py
--- a/bigquery/unit_tests/test__helpers.py
+++ b/bigquery/unit_tests/test__helpers.py
@@ -105,11 +105,11 @@ def test_w_value_other(self):
         self.assertFalse(coerced)
 
 
-class Test_datetime_from_json(unittest.TestCase):
+class Test_timestamp_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
-        from google.cloud.bigquery._helpers import _datetime_from_json
-        return _datetime_from_json(value, field)
+        from google.cloud.bigquery._helpers import _timestamp_from_json
+        return _timestamp_from_json(value, field)
 
     def test_w_none_nullable(self):
         self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
@@ -135,6 +135,27 @@ def test_w_float_value(self):
             _EPOCH + datetime.timedelta(seconds=1, microseconds=234567))
 
 
+class Test_datetime_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _datetime_from_json
+        return _datetime_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field('REQUIRED'))
+
+    def test_w_string_value(self):
+        import datetime
+        coerced = self._call_fut('2016-12-02T18:51:33', object())
+        self.assertEqual(
+            coerced,
+            datetime.datetime(2016, 12, 2, 18, 51, 33))
+
+
 class Test_date_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -411,6 +432,96 @@ def test_w_int64_float64_bool(self):
         self.assertEqual(coerced, expected)
 
 
+class Test_int_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _int_to_json
+        return _int_to_json(value)
+
+    def test_w_int(self):
+        self.assertEqual(self._call_fut(123), '123')
+
+    def test_w_string(self):
+        self.assertEqual(self._call_fut('123'), '123')
+
+
+class Test_float_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _float_to_json
+        return _float_to_json(value)
+
+    def test_w_float(self):
+        self.assertEqual(self._call_fut(1.23), 1.23)
+
+
+class Test_bool_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _bool_to_json
+        return _bool_to_json(value)
+
+    def test_w_true(self):
+        self.assertEqual(self._call_fut(True), 'true')
+
+    def test_w_false(self):
+        self.assertEqual(self._call_fut(False), 'false')
+
+    def test_w_string(self):
+        self.assertEqual(self._call_fut('false'), 'false')
+
+
+class Test_timestamp_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _timestamp_to_json
+        return _timestamp_to_json(value)
+
+    def test_w_float(self):
+        self.assertEqual(self._call_fut(1.234567), 1.234567)
+
+    def test_w_datetime(self):
+        import datetime
+        from google.cloud._helpers import UTC
+        from google.cloud._helpers import _microseconds_from_datetime
+        when = datetime.datetime(2016, 12, 3, 14, 11, 27, tzinfo=UTC)
+        self.assertEqual(self._call_fut(when),
+                         _microseconds_from_datetime(when) / 1e6)
+
+
+class Test_datetime_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _datetime_to_json
+        return _datetime_to_json(value)
+
+    def test_w_string(self):
+        RFC3339 = '2016-12-03T14:14:51Z'
+        self.assertEqual(self._call_fut(RFC3339), RFC3339)
+
+    def test_w_datetime(self):
+        import datetime
+        from google.cloud._helpers import UTC
+        when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=UTC)
+        self.assertEqual(self._call_fut(when), '2016-12-03T14:11:27.123456Z')
+
+
+class Test_date_to_json(unittest.TestCase):
+
+    def _call_fut(self, value):
+        from google.cloud.bigquery._helpers import _date_to_json
+        return _date_to_json(value)
+
+    def test_w_string(self):
+        RFC3339 = '2016-12-03'
+        self.assertEqual(self._call_fut(RFC3339), RFC3339)
+
+    def test_w_datetime(self):
+        import datetime
+        when = datetime.date(2016, 12, 3)
+        self.assertEqual(self._call_fut(when), '2016-12-03')
+
+
 class Test_ConfigurationProperty(unittest.TestCase):
 
     @staticmethod
@@ -668,7 +779,7 @@ def test_to_api_repr_w_name(self):
                 'type': 'INT64',
             },
             'parameterValue': {
-                'value': 123,
+                'value': '123',
             },
         }
         param = self._make_one(name='foo', type_='INT64', value=123)
@@ -680,13 +791,150 @@ def test_to_api_repr_wo_name(self):
                 'type': 'INT64',
             },
             'parameterValue': {
-                'value': 123,
+                'value': '123',
             },
         }
         klass = self._get_target_class()
         param = klass.positional(type_='INT64', value=123)
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test_to_api_repr_w_float(self):
+        EXPECTED = {
+            'parameterType': {
+                'type': 'FLOAT64',
+            },
+            'parameterValue': {
+                'value': 12.345,
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='FLOAT64', value=12.345)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_bool(self):
+        EXPECTED = {
+            'parameterType': {
+                'type': 'BOOL',
+            },
+            'parameterValue': {
+                'value': 'false',
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='BOOL', value=False)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_timestamp_datetime(self):
+        import datetime
+        from google.cloud._helpers import _microseconds_from_datetime
+        now = datetime.datetime.utcnow()
+        seconds = _microseconds_from_datetime(now) / 1.0e6
+        EXPECTED = {
+            'parameterType': {
+                'type': 'TIMESTAMP',
+            },
+            'parameterValue': {
+                'value': seconds,
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='TIMESTAMP', value=now)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_timestamp_micros(self):
+        import datetime
+        from google.cloud._helpers import _microseconds_from_datetime
+        now = datetime.datetime.utcnow()
+        seconds = _microseconds_from_datetime(now) / 1.0e6
+        EXPECTED = {
+            'parameterType': {
+                'type': 'TIMESTAMP',
+            },
+            'parameterValue': {
+                'value': seconds,
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='TIMESTAMP', value=seconds)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_datetime_datetime(self):
+        import datetime
+        from google.cloud._helpers import _datetime_to_rfc3339
+        now = datetime.datetime.utcnow()
+        EXPECTED = {
+            'parameterType': {
+                'type': 'DATETIME',
+            },
+            'parameterValue': {
+                'value': _datetime_to_rfc3339(now),
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='DATETIME', value=now)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_datetime_string(self):
+        import datetime
+        from google.cloud._helpers import _datetime_to_rfc3339
+        now = datetime.datetime.utcnow()
+        now_str = _datetime_to_rfc3339(now)
+        EXPECTED = {
+            'parameterType': {
+                'type': 'DATETIME',
+            },
+            'parameterValue': {
+                'value': now_str,
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='DATETIME', value=now_str)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_date_date(self):
+        import datetime
+        today = datetime.date.today()
+        EXPECTED = {
+            'parameterType': {
+                'type': 'DATE',
+            },
+            'parameterValue': {
+                'value': today.isoformat(),
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='DATE', value=today)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_date_string(self):
+        import datetime
+        today = datetime.date.today()
+        today_str = today.isoformat(),
+        EXPECTED = {
+            'parameterType': {
+                'type': 'DATE',
+            },
+            'parameterValue': {
+                'value': today_str,
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='DATE', value=today_str)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_unknown_type(self):
+        EXPECTED = {
+            'parameterType': {
+                'type': 'UNKNOWN',
+            },
+            'parameterValue': {
+                'value': 'unknown',
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(type_='UNKNOWN', value='unknown')
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
 
 class Test_ArrayQueryParameter(unittest.TestCase):
 
@@ -749,7 +997,7 @@ def test_to_api_repr_w_name(self):
                 'arrayType': 'INT64',
             },
             'parameterValue': {
-                'arrayValues': [1, 2],
+                'arrayValues': ['1', '2'],
             },
         }
         param = self._make_one(name='foo', array_type='INT64', values=[1, 2])
@@ -761,13 +1009,26 @@ def test_to_api_repr_wo_name(self):
                 'arrayType': 'INT64',
             },
             'parameterValue': {
-                'arrayValues': [1, 2],
+                'arrayValues': ['1', '2'],
             },
         }
         klass = self._get_target_class()
         param = klass.positional(array_type='INT64', values=[1, 2])
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test_to_api_repr_w_unknown_type(self):
+        EXPECTED = {
+            'parameterType': {
+                'arrayType': 'UNKNOWN',
+            },
+            'parameterValue': {
+                'arrayValues': ['unknown'],
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.positional(array_type='UNKNOWN', values=['unknown'])
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
 
 class Test_StructQueryParameter(unittest.TestCase):
 
@@ -848,7 +1109,7 @@ def test_to_api_repr_w_name(self):
                 ],
             },
             'parameterValue': {
-                'structValues': {'bar': 123, 'baz': 'abc'},
+                'structValues': {'bar': '123', 'baz': 'abc'},
             },
         }
         sub_1 = self._make_subparam('bar', 'INT64', 123)
@@ -865,7 +1126,7 @@ def test_to_api_repr_wo_name(self):
                 ],
             },
             'parameterValue': {
-                'structValues': {'bar': 123, 'baz': 'abc'},
+                'structValues': {'bar': '123', 'baz': 'abc'},
             },
         }
         sub_1 = self._make_subparam('bar', 'INT64', 123)
diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py
--- a/bigquery/unit_tests/test_job.py
+++ b/bigquery/unit_tests/test_job.py
@@ -1624,7 +1624,7 @@ def test_begin_w_named_query_parameter(self):
                     'type': 'INT64',
                 },
                 'parameterValue': {
-                    'value': 123,
+                    'value': '123',
                 },
             },
         ]
@@ -1674,7 +1674,7 @@ def test_begin_w_positional_query_parameter(self):
                     'type': 'INT64',
                 },
                 'parameterValue': {
-                    'value': 123,
+                    'value': '123',
                 },
             },
         ]
diff --git a/bigquery/unit_tests/test_query.py b/bigquery/unit_tests/test_query.py
--- a/bigquery/unit_tests/test_query.py
+++ b/bigquery/unit_tests/test_query.py
@@ -419,7 +419,7 @@ def test_run_w_named_query_paramter(self):
                     'type': 'INT64',
                 },
                 'parameterValue': {
-                    'value': 123,
+                    'value': '123',
                 },
             },
         ]
@@ -453,7 +453,7 @@ def test_run_w_positional_query_paramter(self):
                     'type': 'INT64',
                 },
                 'parameterValue': {
-                    'value': 123,
+                    'value': '123',
                 },
             },
         ]