Repository: googleapis/python-bigquery

Issue:
Can't use Pandas to upload a REPEATED field (e.g. list of strings)
I am trying to add a list of strings stored in a pandas Dataframe to a BigQuery table with a `REPEATED` field. When running this code:

```
import pandas as pd
from google.cloud import bigquery
from google.oauth2 import service_account

df = pd.DataFrame([{"repeated": ["hi", "hello"], "not_repeated": "a_string"}])

table = bigquery.Table(
    "project.dataset_name.table_name",
    schema=[
        bigquery.SchemaField("repeated", "string", "REPEATED"),
        bigquery.SchemaField("not_repeated", "string", "NULLABLE"),
    ],
)

bigquery_client = bigquery.Client(
    credentials=service_account.Credentials.from_service_account_file(
        "service-account-credentials.json"
    )
)
bigquery_client.insert_rows_from_dataframe(table, df)
```

I get this error:
```
Traceback (most recent call last):
  File "test.py", line 20, in <module>
    bigquery_client.insert_rows_from_dataframe(table, df)
  File "/Users/emmacombes/.local/share/virtualenvs/bq-stats-sAw4GWcD/lib/python3.7/site-packages/google/cloud/bigquery/client.py", line 3433, in insert_rows_from_dataframe
    result = self.insert_rows(table, rows_chunk, selected_fields, **kwargs)
  File "/Users/emmacombes/.local/share/virtualenvs/bq-stats-sAw4GWcD/lib/python3.7/site-packages/google/cloud/bigquery/client.py", line 3381, in insert_rows
    json_rows = [_record_field_to_json(schema, row) for row in rows]
  File "/Users/emmacombes/.local/share/virtualenvs/bq-stats-sAw4GWcD/lib/python3.7/site-packages/google/cloud/bigquery/client.py", line 3381, in <listcomp>
    json_rows = [_record_field_to_json(schema, row) for row in rows]
  File "/Users/emmacombes/.local/share/virtualenvs/bq-stats-sAw4GWcD/lib/python3.7/site-packages/google/cloud/bigquery/_pandas_helpers.py", line 800, in dataframe_to_json_generator
    if pandas.isna(value):
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```
Which stops the execution, and does not allow the code to upload to bigquery. I can confirm that if I run the same code without the list element (aka. `df = pd.DataFrame([{"not_repeated": "a_string"}])`, the error does not occur.

I think this can be traced back to the recently changed line `if pandas.isna(value):` from this [previous PR (use pandas function to check for NaN #750)](https://github.com/googleapis/python-bigquery/pull/750/files) to solve this [previous issue (dataframe_to_json_generator doesn't support pandas.NA type #729 )](https://github.com/googleapis/python-bigquery/issues/729). As evaluating `pandas.isna(value)` on a list will give an array of bools, which can then not be interpreted by the `if` statement.

I can confirm that if I go to an older version of this library before this change was made, the code works.

#### Environment details

  - OS type and version: MacOS BigSur 11.5.2
  - Python version: Python 3.7.5
  - pip version: pip 19.2.3
  - `google-cloud-bigquery` version: 2.24.0



Discussion:
Thanks for reporting this and doing some initial research, it indeed sounds like a regression. 
**Update:** I can confirm the issue. The code sample works in v2.20, but breaks in [v2.21.0](https://github.com/googleapis/python-bigquery/releases/tag/v2.21.0)+. I'll investigate why the tests didn't catch that.

This PR broke it: #750.

Resolving patch:
diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py
--- a/google/cloud/bigquery/_pandas_helpers.py
+++ b/google/cloud/bigquery/_pandas_helpers.py
@@ -844,7 +844,13 @@ def dataframe_to_json_generator(dataframe):
         output = {}
         for column, value in zip(dataframe.columns, row):
             # Omit NaN values.
-            if pandas.isna(value):
+            is_nan = pandas.isna(value)
+
+            # isna() can also return an array-like of bools, but the latter's boolean
+            # value is ambiguous, hence an extra check. An array-like value is *not*
+            # considered a NaN, however.
+            if isinstance(is_nan, bool) and is_nan:
                 continue
             output[column] = value
+
         yield output


Test patch:
diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py
--- a/tests/unit/test__pandas_helpers.py
+++ b/tests/unit/test__pandas_helpers.py
@@ -821,6 +821,41 @@ def test_dataframe_to_json_generator(module_under_test):
     assert list(rows) == expected
 
 
+def test_dataframe_to_json_generator_repeated_field(module_under_test):
+    pytest.importorskip(
+        "pandas",
+        minversion=str(PANDAS_MINIUM_VERSION),
+        reason=(
+            f"Requires `pandas version >= {PANDAS_MINIUM_VERSION}` "
+            "which introduces pandas.NA"
+        ),
+    )
+
+    df_data = [
+        collections.OrderedDict(
+            [("repeated_col", [pandas.NA, 2, None, 4]), ("not_repeated_col", "first")]
+        ),
+        collections.OrderedDict(
+            [
+                ("repeated_col", ["a", "b", mock.sentinel.foo, "d"]),
+                ("not_repeated_col", "second"),
+            ]
+        ),
+    ]
+    dataframe = pandas.DataFrame(df_data)
+
+    rows = module_under_test.dataframe_to_json_generator(dataframe)
+
+    expected = [
+        {"repeated_col": [pandas.NA, 2, None, 4], "not_repeated_col": "first"},
+        {
+            "repeated_col": ["a", "b", mock.sentinel.foo, "d"],
+            "not_repeated_col": "second",
+        },
+    ]
+    assert list(rows) == expected
+
+
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 def test_list_columns_and_indexes_with_named_index(module_under_test):
     df_data = collections.OrderedDict(
@@ -882,7 +917,7 @@ def test_list_columns_and_indexes_with_multiindex(module_under_test):
 def test_dataframe_to_bq_schema_dict_sequence(module_under_test):
     df_data = collections.OrderedDict(
         [
-            ("str_column", [u"hello", u"world"]),
+            ("str_column", ["hello", "world"]),
             ("int_column", [42, 8]),
             ("bool_column", [True, False]),
         ]
@@ -1070,7 +1105,7 @@ def test_dataframe_to_arrow_dict_sequence_schema(module_under_test):
     ]
 
     dataframe = pandas.DataFrame(
-        {"field01": [u"hello", u"world"], "field02": [True, False]}
+        {"field01": ["hello", "world"], "field02": [True, False]}
     )
 
     arrow_table = module_under_test.dataframe_to_arrow(dataframe, dict_schema)
@@ -1139,8 +1174,8 @@ def test_dataframe_to_parquet_compression_method(module_under_test):
 def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test):
     dataframe = pandas.DataFrame(
         data=[
-            {"id": 10, "status": u"FOO", "execution_date": datetime.date(2019, 5, 10)},
-            {"id": 20, "status": u"BAR", "created_at": datetime.date(2018, 9, 12)},
+            {"id": 10, "status": "FOO", "execution_date": datetime.date(2019, 5, 10)},
+            {"id": 20, "status": "BAR", "created_at": datetime.date(2018, 9, 12)},
         ]
     )
 
@@ -1167,8 +1202,8 @@ def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test):
 def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
     dataframe = pandas.DataFrame(
         data=[
-            {"id": 10, "status": u"FOO", "created_at": datetime.date(2019, 5, 10)},
-            {"id": 20, "status": u"BAR", "created_at": datetime.date(2018, 9, 12)},
+            {"id": 10, "status": "FOO", "created_at": datetime.date(2019, 5, 10)},
+            {"id": 20, "status": "BAR", "created_at": datetime.date(2018, 9, 12)},
         ]
     )
 
@@ -1197,8 +1232,8 @@ def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
 def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test):
     dataframe = pandas.DataFrame(
         data=[
-            {"struct_field": {"one": 2}, "status": u"FOO"},
-            {"struct_field": {"two": u"222"}, "status": u"BAR"},
+            {"struct_field": {"one": 2}, "status": "FOO"},
+            {"struct_field": {"two": "222"}, "status": "BAR"},
         ]
     )
 
@@ -1252,7 +1287,7 @@ def test_augment_schema_type_detection_succeeds(module_under_test):
                 "timestamp_field": datetime.datetime(2005, 5, 31, 14, 25, 55),
                 "date_field": datetime.date(2005, 5, 31),
                 "bytes_field": b"some bytes",
-                "string_field": u"some characters",
+                "string_field": "some characters",
                 "numeric_field": decimal.Decimal("123.456"),
                 "bignumeric_field": decimal.Decimal("{d38}.{d38}".format(d38="9" * 38)),
             }
@@ -1312,13 +1347,13 @@ def test_augment_schema_type_detection_fails(module_under_test):
     dataframe = pandas.DataFrame(
         data=[
             {
-                "status": u"FOO",
+                "status": "FOO",
                 "struct_field": {"one": 1},
-                "struct_field_2": {"foo": u"123"},
+                "struct_field_2": {"foo": "123"},
             },
             {
-                "status": u"BAR",
-                "struct_field": {"two": u"111"},
+                "status": "BAR",
+                "struct_field": {"two": "111"},
                 "struct_field_2": {"bar": 27},
             },
         ]
@@ -1351,7 +1386,7 @@ def test_dataframe_to_parquet_dict_sequence_schema(module_under_test):
     ]
 
     dataframe = pandas.DataFrame(
-        {"field01": [u"hello", u"world"], "field02": [True, False]}
+        {"field01": ["hello", "world"], "field02": [True, False]}
     )
 
     write_table_patch = mock.patch.object(