Repository: googleapis/python-bigquery

Issue:
client.insert_rows should fail when inserting non-existing fields
Version
```
google-cloud-bigquery==1.25.0
```

The [`client.insert_rows()`](https://github.com/googleapis/python-bigquery/blob/v1.25.0/google/cloud/bigquery/client.py#L2471) function doesn't fail when inserting non-existing fields whereas the BigQuery API does fail with a message like
```
{
  "kind": "bigquery#tableDataInsertAllResponse",
  "insertErrors": [
    {
      "index": 0,
      "errors": [
        {
          "reason": "invalid",
          "location": "zap",
          "debugInfo": "",
          "message": "no such field."
        }
      ]
    }
  ]
}
```
`insert_rows()` silently drops the additional columns instead.

This happens because `insert_rows()` uses [`_record_field_to_json`](https://github.com/googleapis/python-bigquery/blob/3869e34f3eff6fabca21c397e3f1cbc368ec880e/google/cloud/bigquery/_helpers.py#L409) which only iterates over the list of fields that are provided ignoring all the other fields that are part of the data and `insert_rows()` passes the table's schema as list of fields to `_record_field_to_json`.

This behavior is opposite to the BigQuery API and means we cannot reliably insert data because we're not made aware of changes to the incoming data because there's no failure.
IMHO this behavior is not correct, I think it would be OK if `selected_fields` was provided but it should not silently use the schema to limit which of the fields of the input data are processed and ignore the rest.
I can image there might be cases where one wants to be lenient/ignore all fields that are not part of table so this behavior might have to be an option, possibly combined with `selected_fields`.

P.S. By extension this also applies to the [`client.insert_rows_from_dataframe()`](https://github.com/googleapis/python-bigquery/blob/v1.25.0/google/cloud/bigquery/client.py#L2533) function which uses `client.insert_rows()`.

P.P.S We initially ran into this when using `insert_rows_from_dataframe()` and it was a bit of a search to find where this was going wrong because it's a somewhat indirect chain of `insert_rows_from_dataframe -> insert_rows -> insert_rows_json`.
Why was this long way chosen/added instead of simply using `insert_rows_json(table, df.to_dict(orient="records"))`? It seems a lot simpler and will probably be the workaround we'll implement for now.


Discussion:
@plamut Can you take a look? 
I can confirm the reported behavior, will check how to make the method fail on non-existing fields.

> it's a somewhat indirect chain of insert_rows_from_dataframe -> insert_rows -> insert_rows_json.
Why was this long way chosen/added instead of simply using insert_rows_json(table, df.to_dict(orient="records"))?

IIRC there was a memory concern for large dataframes - `df.to_dict(orient="records")`  would convert the entire dataframe to a list of row dicts, while `insert_rows_from_dataframe()` only processes and inserts `chunk_size` rows at a time.


Resolving patch:
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
--- a/google/cloud/bigquery/_helpers.py
+++ b/google/cloud/bigquery/_helpers.py
@@ -19,6 +19,7 @@
 import datetime
 import decimal
 import re
+import six
 
 from google.cloud._helpers import UTC
 from google.cloud._helpers import _date_from_iso8601_date
@@ -419,9 +420,23 @@ def _record_field_to_json(fields, row_value):
     Returns:
         Mapping[str, Any]: A JSON-serializable dictionary.
     """
-    record = {}
     isdict = isinstance(row_value, dict)
 
+    # If row is passed as a tuple, make the length sanity check to avoid either
+    # uninformative index errors a few lines below or silently omitting some of
+    # the values from the result (we cannot know exactly which fields are missing
+    # or redundant, since we don't have their names).
+    if not isdict and len(row_value) != len(fields):
+        msg = "The number of row fields ({}) does not match schema length ({}).".format(
+            len(row_value), len(fields)
+        )
+        raise ValueError(msg)
+
+    record = {}
+
+    if isdict:
+        processed_fields = set()
+
     for subindex, subfield in enumerate(fields):
         subname = subfield.name
         subvalue = row_value.get(subname) if isdict else row_value[subindex]
@@ -430,6 +445,20 @@ def _record_field_to_json(fields, row_value):
         if subvalue is not None:
             record[subname] = _field_to_json(subfield, subvalue)
 
+        if isdict:
+            processed_fields.add(subname)
+
+    # Unknown fields should not be silently dropped, include them. Since there
+    # is no schema information available for them, include them as strings
+    # to make them JSON-serializable.
+    if isdict:
+        not_processed = set(row_value.keys()) - processed_fields
+
+        for field_name in not_processed:
+            value = row_value[field_name]
+            if value is not None:
+                record[field_name] = six.text_type(value)
+
     return record
 
 


Test patch:
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
--- a/tests/unit/test__helpers.py
+++ b/tests/unit/test__helpers.py
@@ -18,6 +18,7 @@
 import unittest
 
 import mock
+import six
 
 
 class Test_not_null(unittest.TestCase):
@@ -847,6 +848,26 @@ def test_w_non_empty_list(self):
         converted = self._call_fut(fields, original)
         self.assertEqual(converted, {"one": "42", "two": "two"})
 
+    def test_w_list_missing_fields(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = [42]
+
+        with six.assertRaisesRegex(self, ValueError, r".*not match schema length.*"):
+            self._call_fut(fields, original)
+
+    def test_w_list_too_many_fields(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = [42, "two", "three"]
+
+        with six.assertRaisesRegex(self, ValueError, r".*not match schema length.*"):
+            self._call_fut(fields, original)
+
     def test_w_non_empty_dict(self):
         fields = [
             _make_field("INT64", name="one", mode="NULLABLE"),
@@ -890,6 +911,25 @@ def test_w_explicit_none_value(self):
         # None values should be dropped regardless of the field type
         self.assertEqual(converted, {"one": "42"})
 
+    def test_w_dict_unknown_fields(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = {
+            "whoami": datetime.date(2020, 7, 20),
+            "one": 111,
+            "two": "222",
+            "void": None,
+        }
+
+        converted = self._call_fut(fields, original)
+
+        # Unknown fields should be included (if not None), but converted as strings.
+        self.assertEqual(
+            converted, {"whoami": "2020-07-20", "one": "111", "two": "222"},
+        )
+
 
 class Test_field_to_json(unittest.TestCase):
     def _call_fut(self, field, value):