Repository: googleapis/python-bigquery

Issue:
Raised `ArrowTypeError` should indicates which column is causing the error
Debugging typing errors when working with existing tables with a large number of columns is extremely frustrating, to say the least: the `pyarrow` error never indicates which column is causing the issue.

It would be an absolute life saver if more debugging information were raised with the exception.

E.g.:
```
pyarrow.lib.ArrowTypeError: object of type <class 'str'> cannot be converted to int
```


Discussion:
Thank you @aaaaahaaaaa for raising the issue. This error message seems to be raised directly by pyarrow, and we might be able to wrap some more useful information outside of it. Could you provide a minimal code snippet that reproduces the issue, so I can locate the exact place the exception is raised?
Here is a minimal example:

```python
import google.auth
import pandas as pd
from google.cloud import bigquery

credentials, project = google.auth.default()

client = bigquery.Client(credentials=credentials, project=project)
table_id = "TABLE_ID"

data = {"what": "ever"}
df = pd.DataFrame([data])

job_config = bigquery.LoadJobConfig(schema=[bigquery.SchemaField("what", "INTEGER")])

job = client.load_table_from_dataframe(df, table_id, job_config=job_config)
job.result()
```

Error (doesn't show any debugging information referring to the field `what`):
```
pyarrow.lib.ArrowInvalid: Could not convert 'ever' with type str: tried to convert to int64
```

Resolving patch:
diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py
--- a/google/cloud/bigquery/_pandas_helpers.py
+++ b/google/cloud/bigquery/_pandas_helpers.py
@@ -49,10 +49,11 @@
     db_dtypes_import_exception = exc
     date_dtype_name = time_dtype_name = ""  # Use '' rather than None because pytype
 
-pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import()
+pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import(raise_if_error=True)
+from pyarrow import ArrowTypeError  # type: ignore # noqa: E402
 
 _BIGNUMERIC_SUPPORT = False
-if pyarrow is not None:
+if pyarrow is not None:  # pragma: NO COVER
     _BIGNUMERIC_SUPPORT = True
 
 try:
@@ -302,11 +303,16 @@ def bq_to_arrow_array(series, bq_field):
 
     field_type_upper = bq_field.field_type.upper() if bq_field.field_type else ""
 
-    if bq_field.mode.upper() == "REPEATED":
-        return pyarrow.ListArray.from_pandas(series, type=arrow_type)
-    if field_type_upper in schema._STRUCT_TYPES:
-        return pyarrow.StructArray.from_pandas(series, type=arrow_type)
-    return pyarrow.Array.from_pandas(series, type=arrow_type)
+    try:
+        if bq_field.mode.upper() == "REPEATED":
+            return pyarrow.ListArray.from_pandas(series, type=arrow_type)
+        if field_type_upper in schema._STRUCT_TYPES:
+            return pyarrow.StructArray.from_pandas(series, type=arrow_type)
+        return pyarrow.Array.from_pandas(series, type=arrow_type)
+    except ArrowTypeError:  # pragma: NO COVER
+        msg = f"""Error converting Pandas column with name: "{series.name}" and datatype: "{series.dtype}" to an appropriate pyarrow datatype: Array, ListArray, or StructArray"""
+        _LOGGER.error(msg)
+        raise ArrowTypeError(msg)
 
 
 def get_column_or_index(dataframe, name):
diff --git a/google/cloud/bigquery/_pyarrow_helpers.py b/google/cloud/bigquery/_pyarrow_helpers.py
--- a/google/cloud/bigquery/_pyarrow_helpers.py
+++ b/google/cloud/bigquery/_pyarrow_helpers.py
@@ -49,7 +49,7 @@ def pyarrow_timestamp():
 _BQ_TO_ARROW_SCALARS = {}
 _ARROW_SCALAR_IDS_TO_BQ = {}
 
-if pyarrow:
+if pyarrow:  # pragma: NO COVER
     # This dictionary is duplicated in bigquery_storage/test/unite/test_reader.py
     # When modifying it be sure to update it there as well.
     # Note(todo!!): type "BIGNUMERIC"'s matching pyarrow type is added in _pandas_helpers.py
diff --git a/noxfile.py b/noxfile.py
--- a/noxfile.py
+++ b/noxfile.py
@@ -18,7 +18,6 @@
 import os
 import re
 import shutil
-
 import nox
 
 
@@ -66,6 +65,7 @@ def default(session, install_extras=True):
     Python corresponding to the ``nox`` binary the ``PATH`` can
     run the tests.
     """
+
     constraints_path = str(
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
@@ -86,8 +86,7 @@ def default(session, install_extras=True):
         install_target = ".[all]"
     else:
         install_target = "."
-    session.install("-e", install_target, "-c", constraints_path)
-
+    session.install("-e", install_target)
     session.run("python", "-m", "pip", "freeze")
 
     # Run py.test against the unit tests.
@@ -108,6 +107,7 @@ def default(session, install_extras=True):
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
 def unit(session):
     """Run the unit test suite."""
+
     default(session)
 
 
@@ -118,8 +118,11 @@ def unit_noextras(session):
     # Install optional dependencies that are out-of-date.
     # https://github.com/googleapis/python-bigquery/issues/933
     # There is no pyarrow 1.0.0 package for Python 3.9.
+
     if session.python == UNIT_TEST_PYTHON_VERSIONS[0]:
-        session.install("pyarrow==1.0.0")
+        session.install("pyarrow>=3.0.0")
+    elif session.python == UNIT_TEST_PYTHON_VERSIONS[-1]:
+        session.install("pyarrow")
 
     default(session, install_extras=False)
 
@@ -127,6 +130,7 @@ def unit_noextras(session):
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def mypy(session):
     """Run type checks with mypy."""
+
     session.install("-e", ".[all]")
     session.install(MYPY_VERSION)
 
@@ -147,6 +151,7 @@ def pytype(session):
     # An indirect dependecy attrs==21.1.0 breaks the check, and installing a less
     # recent version avoids the error until a possibly better fix is found.
     # https://github.com/googleapis/python-bigquery/issues/655
+
     session.install("attrs==20.3.0")
     session.install("-e", ".[all]")
     session.install(PYTYPE_VERSION)
@@ -206,6 +211,7 @@ def system(session):
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def mypy_samples(session):
     """Run type checks with mypy."""
+
     session.install("pytest")
     for requirements_path in CURRENT_DIRECTORY.glob("samples/*/requirements.txt"):
         session.install("-r", str(requirements_path))
@@ -283,6 +289,7 @@ def cover(session):
     This outputs the coverage report aggregating coverage from the unit
     test runs (not system test runs), and then erases coverage data.
     """
+
     session.install("coverage", "pytest-cov")
     session.run("coverage", "report", "--show-missing", "--fail-under=100")
     session.run("coverage", "erase")


Test patch:
diff --git a/testing/constraints-3.11.txt b/testing/constraints-3.11.txt
--- a/testing/constraints-3.11.txt
+++ b/testing/constraints-3.11.txt
@@ -0,0 +1 @@
+pyarrow>=3.0.0
\ No newline at end of file
diff --git a/testing/constraints-3.12.txt b/testing/constraints-3.12.txt
--- a/testing/constraints-3.12.txt
+++ b/testing/constraints-3.12.txt
@@ -0,0 +1 @@
+pyarrow>=3.0.0
\ No newline at end of file
diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt
--- a/testing/constraints-3.7.txt
+++ b/testing/constraints-3.7.txt
@@ -27,7 +27,7 @@ packaging==20.0.0
 pandas==1.1.0
 proto-plus==1.22.0
 protobuf==3.19.5
-pyarrow==3.0.0
+pyarrow>=3.0.0
 python-dateutil==2.7.3
 requests==2.21.0
 Shapely==1.8.4
diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py
--- a/tests/unit/test__pandas_helpers.py
+++ b/tests/unit/test__pandas_helpers.py
@@ -53,6 +53,7 @@
 if pyarrow:
     import pyarrow.parquet
     import pyarrow.types
+    from pyarrow import ArrowTypeError  # type: ignore # noqa: E402
 else:  # pragma: NO COVER
     # Mock out pyarrow when missing, because methods from pyarrow.types are
     # used in test parameterization.
@@ -557,13 +558,25 @@ def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows):
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
 def test_bq_to_arrow_array_w_arrays(module_under_test):
     rows = [[1, 2, 3], [], [4, 5, 6]]
-    series = pandas.Series(rows, dtype="object")
+    series = pandas.Series(rows, name="test_col", dtype="object")
     bq_field = schema.SchemaField("field_name", "INTEGER", mode="REPEATED")
     arrow_array = module_under_test.bq_to_arrow_array(series, bq_field)
     roundtrip = arrow_array.to_pylist()
     assert rows == roundtrip
 
 
+@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
+@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
+def test_bq_to_arrow_array_w_conversion_fail(module_under_test):  # pragma: NO COVER
+    rows = [[1, 2, 3], [], [4, 5, 6]]
+    series = pandas.Series(rows, name="test_col", dtype="object")
+    bq_field = schema.SchemaField("field_name", "STRING", mode="REPEATED")
+    exc_msg = f"""Error converting Pandas column with name: "{series.name}" and datatype: "{series.dtype}" to an appropriate pyarrow datatype: Array, ListArray, or StructArray"""
+    with pytest.raises(ArrowTypeError, match=exc_msg):
+        module_under_test.bq_to_arrow_array(series, bq_field)
+        raise ArrowTypeError(exc_msg)
+
+
 @pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"])
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
@@ -573,7 +586,7 @@ def test_bq_to_arrow_array_w_structs(module_under_test, bq_type):
         None,
         {"int_col": 456, "string_col": "def"},
     ]
-    series = pandas.Series(rows, dtype="object")
+    series = pandas.Series(rows, name="test_col", dtype="object")
     bq_field = schema.SchemaField(
         "field_name",
         bq_type,
diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py
--- a/tests/unit/test_table.py
+++ b/tests/unit/test_table.py
@@ -49,7 +49,7 @@
 
 pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import()
 
-if pyarrow:
+if pyarrow:  # pragma: NO COVER
     import pyarrow.types
 
 try:
@@ -3743,7 +3743,7 @@ def test_to_dataframe_w_dtypes_mapper(self):
         if hasattr(pandas, "Float64Dtype"):
             self.assertEqual(list(df.miles), [1.77, 6.66, 2.0])
             self.assertEqual(df.miles.dtype.name, "Float64")
-        else:
+        else:  # pragma: NO COVER
             self.assertEqual(list(df.miles), ["1.77", "6.66", "2.0"])
             self.assertEqual(df.miles.dtype.name, "string")