Repository: googleapis/google-cloud-python

Issue:
BigQuery: KeyError inserting data with missing value into table with nullable field
I recently updated the google-cloud-bigquery python package and I am getting a **KeyError** when trying to insert a row with a **missing value** into a table with a **nullable field**.
In previous version (I tested it on version 1.5.0) the missing field was automatically filled with a _null_ value.

#### Environment details

OS: Ubuntu 18.04
Python version:
```
$ python --version
Python 3.6.7
```
Package version: 
```
$ pip show google-cloud-bigquery
Name: google-cloud-bigquery
Version: 1.9.0
```
Note: It works on version 1.5.0

#### Steps to reproduce

  1. Create table with nullable field
  2. Insert row with missing value (provide row as dictionary, with a missing key:value)

#### Code example

```python
from google.cloud import bigquery

client = bigquery.Client(project='my_project')

dataset_ref = client.dataset('my_dataset')
schema = [
    bigquery.SchemaField("user_id", "STRING", mode="REQUIRED"),
    bigquery.SchemaField("name", "STRING", mode="NULLABLE"), # NULLABLE
]
table_ref = dataset_ref.table("my_table")
table = bigquery.Table(table_ref, schema=schema)
table = client.create_table(table)

rows_to_insert = [
    {'user_id': 1111, 'name': 'peter'},  # OK
    {'user_id': 2222, 'name': 'anna'}  # OK
]

errors = client.insert_rows(table, rows_to_insert)
assert errors == []

rows_to_insert = [
    {'user_id': 3333, 'name': 'john'},
    {'user_id': 4444} # FAILS
]

errors = client.insert_rows(table, rows_to_insert) # FAILS HERE
assert errors == []
```

#### Stack trace
```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-22-a6f1bda68730> in <module>
     26 ]
     27 
---> 28 errors = client.insert_rows(table, rows_to_insert) # FAILS HERE
     29 assert errors == []

~/.local/lib/python3.6/site-packages/google/cloud/bigquery/client.py in insert_rows(self, table, rows, selected_fields, **kwargs)
   1495             raise TypeError("table should be Table or TableReference")
   1496 
-> 1497         json_rows = [_record_field_to_json(schema, row) for row in rows]
   1498 
   1499         return self.insert_rows_json(table, json_rows, **kwargs)

~/.local/lib/python3.6/site-packages/google/cloud/bigquery/client.py in <listcomp>(.0)
   1495             raise TypeError("table should be Table or TableReference")
   1496 
-> 1497         json_rows = [_record_field_to_json(schema, row) for row in rows]
   1498 
   1499         return self.insert_rows_json(table, json_rows, **kwargs)

~/.local/lib/python3.6/site-packages/google/cloud/bigquery/_helpers.py in _record_field_to_json(fields, row_value)
    399     for subindex, subfield in enumerate(fields):
    400         subname = subfield.name
--> 401         subvalue = row_value[subname] if isdict else row_value[subindex]
    402         record[subname] = _field_to_json(subfield, subvalue)
    403     return record

KeyError: 'name'
```



Discussion:
/cc @adamf 

This change was introduced in e633ffdb96acebbd8c87d1f52edeb2432f6ac7e0, which fixed parsing nested schemas.

I think the change is as simple as using `row_value.get(subname)` instead of `row_value[subname]` in line 401.  Of course, we should add a test which exercises the missing value logic.

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -398,7 +398,10 @@ def _record_field_to_json(fields, row_value):
 
     for subindex, subfield in enumerate(fields):
         subname = subfield.name
-        subvalue = row_value[subname] if isdict else row_value[subindex]
+        if isdict:
+            subvalue = row_value.get(subname)
+        else:
+            subvalue = row_value[subindex]
         record[subname] = _field_to_json(subfield, subvalue)
     return record
 


Test patch:
diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py
--- a/bigquery/tests/unit/test__helpers.py
+++ b/bigquery/tests/unit/test__helpers.py
@@ -776,6 +776,127 @@ def test_w_datetime(self):
         self.assertEqual(self._call_fut(when), "12:13:41")
 
 
+def _make_field(field_type, mode="NULLABLE", name="testing", fields=()):
+    from google.cloud.bigquery.schema import SchemaField
+
+    return SchemaField(name=name, field_type=field_type, mode=mode, fields=fields)
+
+
+class Test_scalar_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _scalar_field_to_json
+
+        return _scalar_field_to_json(field, value)
+
+    def test_w_unknown_field_type(self):
+        field = _make_field("UNKNOWN")
+        original = object()
+        converted = self._call_fut(field, original)
+        self.assertIs(converted, original)
+
+    def test_w_known_field_type(self):
+        field = _make_field("INT64")
+        original = 42
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, str(original))
+
+
+class Test_repeated_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _repeated_field_to_json
+
+        return _repeated_field_to_json(field, value)
+
+    def test_w_empty(self):
+        field = _make_field("INT64", mode="REPEATED")
+        original = []
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, original)
+        self.assertEqual(field.mode, "REPEATED")
+
+    def test_w_non_empty(self):
+        field = _make_field("INT64", mode="REPEATED")
+        original = [42]
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, [str(value) for value in original])
+        self.assertEqual(field.mode, "REPEATED")
+
+
+class Test_record_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _record_field_to_json
+
+        return _record_field_to_json(field, value)
+
+    def test_w_empty(self):
+        fields = []
+        original = []
+        converted = self._call_fut(fields, original)
+        self.assertEqual(converted, {})
+
+    def test_w_non_empty_list(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = [42, "two"]
+        converted = self._call_fut(fields, original)
+        self.assertEqual(converted, {"one": "42", "two": "two"})
+
+    def test_w_non_empty_dict(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = {"one": 42, "two": "two"}
+        converted = self._call_fut(fields, original)
+        self.assertEqual(converted, {"one": "42", "two": "two"})
+
+    def test_w_missing_nullable(self):
+        fields = [
+            _make_field("INT64", name="one", mode="NULLABLE"),
+            _make_field("STRING", name="two", mode="NULLABLE"),
+        ]
+        original = {"one": 42}
+        converted = self._call_fut(fields, original)
+        self.assertEqual(converted, {"one": "42", "two": None})
+
+
+class Test_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _field_to_json
+
+        return _field_to_json(field, value)
+
+    def test_w_none(self):
+        field = _make_field("INT64")
+        original = None
+        converted = self._call_fut(field, original)
+        self.assertIsNone(converted)
+
+    def test_w_repeated(self):
+        field = _make_field("INT64", mode="REPEATED")
+        original = [42, 17]
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, [str(value) for value in original])
+
+    def test_w_record(self):
+        subfields = [
+            _make_field("INT64", name="one"),
+            _make_field("STRING", name="two"),
+        ]
+        field = _make_field("RECORD", fields=subfields)
+        original = {"one": 42, "two": "two"}
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, {"one": "42", "two": "two"})
+
+    def test_w_scalar(self):
+        field = _make_field("INT64")
+        original = 42
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, str(original))
+
+
 class Test_snake_to_camel_case(unittest.TestCase):
     def _call_fut(self, value):
         from google.cloud.bigquery._helpers import _snake_to_camel_case