Repository: googleapis/python-bigquery

Issue:
DB-API depends on pyarrow when decimal query parameters are used
It seems we introduced a hard dependency on pyarrow to detect data types of query parameters in the DB-API module. While `pyarrow` is recommended as it's needed to use the BigQuery Storage API, I don't think we want to depend on it for basic functionality.

I believe we added this hard dependency here: https://github.com/googleapis/python-bigquery/pull/527/files#diff-2221cd9b6b85f2ed1f943c474e8a8628e6ca28ec9a96c0aa5873cd6a66826bdeR193

We use `pyarrow` to disambiguate between `NUMERIC` and `BIGNUMERIC`. I'm not seeing a good way to check the precision / scale in the [decimal](https://docs.python.org/3/library/decimal.html) type, so in the case that `pyarrow` is not available, we should guess the type is `NUMERIC`.


#### Stack trace

```
__________________________________________________________________________________ test_dml ___________________________________________________________________________________
test/test_sqlalchemy_bigquery.py:430: in test_dml
    engine.execute(table_dml.insert(ONE_ROW_CONTENTS_DML))
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:2235: in execute
    return connection.execute(statement, *multiparams, **params)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1011: in execute
    return meth(self, multiparams, params)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/sql/elements.py:298: in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1124: in _execute_clauseelement
    ret = self._execute_context(
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1316: in _execute_context
    self._handle_dbapi_exception(
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1514: in _handle_dbapi_exception
    util.raise_(exc_info[1], with_traceback=exc_info[2])
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/util/compat.py:182: in raise_
    raise exception
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1276: in _execute_context
    self.dialect.do_execute(
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/sqlalchemy/engine/default.py:608: in do_execute
    cursor.execute(statement, parameters)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/_helpers.py:263: in with_closed_check
    return method(self, *args, **kwargs)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/cursor.py:165: in execute
    query_parameters = _helpers.to_query_parameters(parameters)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/_helpers.py:168: in to_query_parameters
    return to_query_parameters_dict(parameters)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/_helpers.py:147: in to_query_parameters_dict
    param = scalar_to_query_parameter(value, name=name)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/_helpers.py:51: in scalar_to_query_parameter
    parameter_type = bigquery_scalar_type(value)
../../miniconda3/envs/pybigquery/lib/python3.9/site-packages/google/cloud/bigquery/dbapi/_helpers.py:193: in bigquery_scalar_type
    scalar_object = pyarrow.scalar(value)
E   AttributeError: 'NoneType' object has no attribute 'scalar'
```




Discussion:

Resolving patch:
diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py
--- a/google/cloud/bigquery/dbapi/_helpers.py
+++ b/google/cloud/bigquery/dbapi/_helpers.py
@@ -19,16 +19,15 @@
 import functools
 import numbers
 
-try:
-    import pyarrow
-except ImportError:  # pragma: NO COVER
-    pyarrow = None
-
 from google.cloud import bigquery
 from google.cloud.bigquery import table
 from google.cloud.bigquery.dbapi import exceptions
 
 
+_NUMERIC_SERVER_MIN = decimal.Decimal("-9.9999999999999999999999999999999999999E+28")
+_NUMERIC_SERVER_MAX = decimal.Decimal("9.9999999999999999999999999999999999999E+28")
+
+
 def scalar_to_query_parameter(value, name=None):
     """Convert a scalar value into a query parameter.
 
@@ -189,12 +188,20 @@ def bigquery_scalar_type(value):
     elif isinstance(value, numbers.Real):
         return "FLOAT64"
     elif isinstance(value, decimal.Decimal):
-        # We check for NUMERIC before BIGNUMERIC in order to support pyarrow < 3.0.
-        scalar_object = pyarrow.scalar(value)
-        if isinstance(scalar_object, pyarrow.Decimal128Scalar):
+        vtuple = value.as_tuple()
+        # NUMERIC values have precision of 38 (number of digits) and scale of 9 (number
+        # of fractional digits), and their max absolute value must be strictly smaller
+        # than 1.0E+29.
+        # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types
+        if (
+            len(vtuple.digits) <= 38  # max precision: 38
+            and vtuple.exponent >= -9  # max scale: 9
+            and _NUMERIC_SERVER_MIN <= value <= _NUMERIC_SERVER_MAX
+        ):
             return "NUMERIC"
         else:
             return "BIGNUMERIC"
+
     elif isinstance(value, str):
         return "STRING"
     elif isinstance(value, bytes):


Test patch:
diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py
--- a/tests/unit/test_dbapi__helpers.py
+++ b/tests/unit/test_dbapi__helpers.py
@@ -25,7 +25,6 @@
 
 import google.cloud._helpers
 from google.cloud.bigquery import table
-from google.cloud.bigquery._pandas_helpers import _BIGNUMERIC_SUPPORT
 from google.cloud.bigquery.dbapi import _helpers
 from google.cloud.bigquery.dbapi import exceptions
 from tests.unit.helpers import _to_pyarrow
@@ -39,9 +38,8 @@ def test_scalar_to_query_parameter(self):
             (123, "INT64"),
             (-123456789, "INT64"),
             (1.25, "FLOAT64"),
-            (decimal.Decimal("1.25"), "NUMERIC"),
             (b"I am some bytes", "BYTES"),
-            (u"I am a string", "STRING"),
+            ("I am a string", "STRING"),
             (datetime.date(2017, 4, 1), "DATE"),
             (datetime.time(12, 34, 56), "TIME"),
             (datetime.datetime(2012, 3, 4, 5, 6, 7), "DATETIME"),
@@ -51,14 +49,17 @@ def test_scalar_to_query_parameter(self):
                 ),
                 "TIMESTAMP",
             ),
+            (decimal.Decimal("1.25"), "NUMERIC"),
+            (decimal.Decimal("9.9999999999999999999999999999999999999E+28"), "NUMERIC"),
+            (decimal.Decimal("1.0E+29"), "BIGNUMERIC"),  # more than max NUMERIC value
+            (decimal.Decimal("1.123456789"), "NUMERIC"),
+            (decimal.Decimal("1.1234567891"), "BIGNUMERIC"),  # scale > 9
+            (decimal.Decimal("12345678901234567890123456789.012345678"), "NUMERIC"),
+            (
+                decimal.Decimal("12345678901234567890123456789012345678"),
+                "BIGNUMERIC",  # larger than max NUMERIC value, despite precision <=38
+            ),
         ]
-        if _BIGNUMERIC_SUPPORT:
-            expected_types.append(
-                (
-                    decimal.Decimal("1.1234567890123456789012345678901234567890"),
-                    "BIGNUMERIC",
-                )
-            )
 
         for value, expected_type in expected_types:
             msg = "value: {} expected_type: {}".format(value, expected_type)
@@ -71,6 +72,33 @@ def test_scalar_to_query_parameter(self):
             self.assertEqual(named_parameter.type_, expected_type, msg=msg)
             self.assertEqual(named_parameter.value, value, msg=msg)
 
+    def test_decimal_to_query_parameter(self):  # TODO: merge with previous test
+
+        expected_types = [
+            (decimal.Decimal("9.9999999999999999999999999999999999999E+28"), "NUMERIC"),
+            (decimal.Decimal("1.0E+29"), "BIGNUMERIC"),  # more than max value
+            (decimal.Decimal("1.123456789"), "NUMERIC"),
+            (decimal.Decimal("1.1234567891"), "BIGNUMERIC"),  # scale > 9
+            (decimal.Decimal("12345678901234567890123456789.012345678"), "NUMERIC"),
+            (
+                decimal.Decimal("12345678901234567890123456789012345678"),
+                "BIGNUMERIC",  # larger than max size, even if precision <=38
+            ),
+        ]
+
+        for value, expected_type in expected_types:
+            msg = f"value: {value} expected_type: {expected_type}"
+
+            parameter = _helpers.scalar_to_query_parameter(value)
+            self.assertIsNone(parameter.name, msg=msg)
+            self.assertEqual(parameter.type_, expected_type, msg=msg)
+            self.assertEqual(parameter.value, value, msg=msg)
+
+            named_parameter = _helpers.scalar_to_query_parameter(value, name="myvar")
+            self.assertEqual(named_parameter.name, "myvar", msg=msg)
+            self.assertEqual(named_parameter.type_, expected_type, msg=msg)
+            self.assertEqual(named_parameter.value, value, msg=msg)
+
     def test_scalar_to_query_parameter_w_unexpected_type(self):
         with self.assertRaises(exceptions.ProgrammingError):
             _helpers.scalar_to_query_parameter(value={"a": "dictionary"})
@@ -89,8 +117,9 @@ def test_array_to_query_parameter_valid_argument(self):
             ([123, -456, 0], "INT64"),
             ([1.25, 2.50], "FLOAT64"),
             ([decimal.Decimal("1.25")], "NUMERIC"),
+            ([decimal.Decimal("{d38}.{d38}".format(d38="9" * 38))], "BIGNUMERIC"),
             ([b"foo", b"bar"], "BYTES"),
-            ([u"foo", u"bar"], "STRING"),
+            (["foo", "bar"], "STRING"),
             ([datetime.date(2017, 4, 1), datetime.date(2018, 4, 1)], "DATE"),
             ([datetime.time(12, 34, 56), datetime.time(10, 20, 30)], "TIME"),
             (
@@ -113,11 +142,6 @@ def test_array_to_query_parameter_valid_argument(self):
             ),
         ]
 
-        if _BIGNUMERIC_SUPPORT:
-            expected_types.append(
-                ([decimal.Decimal("{d38}.{d38}".format(d38="9" * 38))], "BIGNUMERIC")
-            )
-
         for values, expected_type in expected_types:
             msg = "value: {} expected_type: {}".format(values, expected_type)
             parameter = _helpers.array_to_query_parameter(values)
@@ -134,7 +158,7 @@ def test_array_to_query_parameter_empty_argument(self):
             _helpers.array_to_query_parameter([])
 
     def test_array_to_query_parameter_unsupported_sequence(self):
-        unsupported_iterables = [{10, 20, 30}, u"foo", b"bar", bytearray([65, 75, 85])]
+        unsupported_iterables = [{10, 20, 30}, "foo", b"bar", bytearray([65, 75, 85])]
         for iterable in unsupported_iterables:
             with self.assertRaises(exceptions.ProgrammingError):
                 _helpers.array_to_query_parameter(iterable)
@@ -144,7 +168,7 @@ def test_array_to_query_parameter_sequence_w_invalid_elements(self):
             _helpers.array_to_query_parameter([object(), 2, 7])
 
     def test_to_query_parameters_w_dict(self):
-        parameters = {"somebool": True, "somestring": u"a-string-value"}
+        parameters = {"somebool": True, "somestring": "a-string-value"}
         query_parameters = _helpers.to_query_parameters(parameters)
         query_parameter_tuples = []
         for param in query_parameters:
@@ -154,7 +178,7 @@ def test_to_query_parameters_w_dict(self):
             sorted(
                 [
                     ("somebool", "BOOL", True),
-                    ("somestring", "STRING", u"a-string-value"),
+                    ("somestring", "STRING", "a-string-value"),
                 ]
             ),
         )
@@ -177,14 +201,14 @@ def test_to_query_parameters_w_dict_dict_param(self):
             _helpers.to_query_parameters(parameters)
 
     def test_to_query_parameters_w_list(self):
-        parameters = [True, u"a-string-value"]
+        parameters = [True, "a-string-value"]
         query_parameters = _helpers.to_query_parameters(parameters)
         query_parameter_tuples = []
         for param in query_parameters:
             query_parameter_tuples.append((param.name, param.type_, param.value))
         self.assertSequenceEqual(
             sorted(query_parameter_tuples),
-            sorted([(None, "BOOL", True), (None, "STRING", u"a-string-value")]),
+            sorted([(None, "BOOL", True), (None, "STRING", "a-string-value")]),
         )
 
     def test_to_query_parameters_w_list_array_param(self):