Repository: googleapis/python-bigquery

Issue:
Deserializing JSON subfields within structs fails
#### Environment details

  - OS type and version: MacOS Ventura 13.1
  - Python version: 3.9.5
  - pip version: 22.3.1
  - `google-cloud-bigquery` version: 3.5.0

#### Steps to reproduce

1. Create table with record field containing a JSON subfield
2. Query record field and try to read row from query result

#### Code example

```python
from google.cloud import bigquery
bigquery.Client().query("SELECT record_with_json_subfield.* FROM <table>").result().__next__()
```

#### Stack trace
```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "<stdin>", line 1, in <listcomp>
  File "lib/python3.9/site-packages/google/api_core/page_iterator.py", line 209, in _items_iter
    for item in page:
  File "lib/python3.9/site-packages/google/api_core/page_iterator.py", line 131, in __next__
    result = self._item_to_value(self._parent, item)
  File "lib/python3.9/site-packages/google/cloud/bigquery/table.py", line 2710, in _item_to_row
    _helpers._row_tuple_from_json(resource, iterator.schema),
  File "lib/python3.9/site-packages/google/cloud/bigquery/_helpers.py", line 435, in _row_tuple_from_json
    row_data.append(_field_from_json(cell["v"], field))
  File "lib/python3.9/site-packages/google/cloud/bigquery/_helpers.py", line 411, in _field_from_json
    return converter(resource, field)
  File "lib/python3.9/site-packages/google/cloud/bigquery/_helpers.py", line 368, in _record_from_json
    converter = _CELLDATA_FROM_JSON[subfield.field_type]
KeyError: 'JSON'
```


Discussion:
I just encountered the same issue on 2.34.4.
I ran this query using version 3.10.0 with no issues.
I believe it happens with nested jsons. I encountered it on version 3.10.0 as well.
Here is a hotfix that was working for me:
```
from google.cloud import bigquery

import json

def _json_from_json(value, _):
    """NOOP string -> string coercion"""
    return json.loads(value)


bigquery._helpers._CELLDATA_FROM_JSON['JSON'] = _json_from_json
```
Could you apply this hotfix to the stable version? It seems like to work
The issue is happening also on earlier versions of the SDK. Tested on 2.20.0 and 2.34.1.

Resolving patch:
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
--- a/google/cloud/bigquery/_helpers.py
+++ b/google/cloud/bigquery/_helpers.py
@@ -17,6 +17,7 @@
 import base64
 import datetime
 import decimal
+import json
 import math
 import re
 import os
@@ -412,6 +413,12 @@ def _time_to_json(value):
     return value
 
 
+def _json_from_json(value, field):
+    """Coerce 'value' to a pythonic JSON representation, if set or not nullable."""
+    if _not_null(value, field):
+        return json.loads(value)
+
+
 # Converters used for scalar values marshalled as row data.
 _SCALAR_VALUE_TO_JSON_ROW = {
     "INTEGER": _int_to_json,
@@ -427,6 +434,7 @@ def _time_to_json(value):
     "DATETIME": _datetime_to_json,
     "DATE": _date_to_json,
     "TIME": _time_to_json,
+    "JSON": _json_from_json,
     # Make sure DECIMAL and BIGDECIMAL are handled, even though
     # requests for them should be converted to NUMERIC.  Better safe
     # than sorry.
diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py
--- a/google/cloud/bigquery/query.py
+++ b/google/cloud/bigquery/query.py
@@ -469,7 +469,7 @@ def to_api_repr(self) -> dict:
         value = self.value
         converter = _SCALAR_VALUE_TO_JSON_PARAM.get(self.type_)
         if converter is not None:
-            value = converter(value)
+            value = converter(value)  # type: ignore
         resource: Dict[str, Any] = {
             "parameterType": {"type": self.type_},
             "parameterValue": {"value": value},
@@ -626,7 +626,7 @@ def to_api_repr(self) -> dict:
 
             converter = _SCALAR_VALUE_TO_JSON_PARAM.get(a_type["type"])
             if converter is not None:
-                values = [converter(value) for value in values]
+                values = [converter(value) for value in values]  # type: ignore
             a_values = [{"value": value} for value in values]
 
         resource = {
@@ -775,7 +775,7 @@ def to_api_repr(self) -> dict:
                 s_types[name] = {"name": name, "type": {"type": type_}}
                 converter = _SCALAR_VALUE_TO_JSON_PARAM.get(type_)
                 if converter is not None:
-                    value = converter(value)
+                    value = converter(value)  # type: ignore
                 values[name] = {"value": value}
 
         resource = {


Test patch:
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
--- a/tests/unit/test__helpers.py
+++ b/tests/unit/test__helpers.py
@@ -58,6 +58,24 @@ def test_w_float_value(self):
         self.assertEqual(coerced, 42)
 
 
+class Test_json_from_json(unittest.TestCase):
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery._helpers import _json_from_json
+
+        return _json_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field("NULLABLE")))
+
+    def test_w_none_required(self):
+        with self.assertRaises(TypeError):
+            self._call_fut(None, _Field("REQUIRED"))
+
+    def test_w_string_value(self):
+        coerced = self._call_fut('{"foo": true}', object())
+        self.assertEqual(coerced, {"foo": True})
+
+
 class Test_float_from_json(unittest.TestCase):
     def _call_fut(self, value, field):
         from google.cloud.bigquery._helpers import _float_from_json