Repository: googleapis/google-cloud-python

Issue:
Support (non-legacy) SQL nested data types
`STRUCT<t>` and `ARRAY<t>` seem to be the types.

See #2342. h/t to @c0b for filing.

---

From #2585: [Sample app that reproduces this](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/176f161cae3d4f01b01456a6e403faf6645414a1/bigquery/cloud-client/simple_app.py)

Stacktrace:

``` python
Traceback (most recent call last):
  File "simple_app.py", line 57, in <module>
    query_shakespeare()
  File "simple_app.py", line 47, in query_shakespeare
    page_token=page_token)
  File "lib/python2.7/site-packages/google/cloud/bigquery/query.py", line 401, in fetch_data
    rows_data = _rows_from_json(response.get('rows', ()), self.schema)
  File "lib/python2.7/site-packages/google/cloud/bigquery/_helpers.py", line 98, in _rows_from_json
    for item in cell['v']])
  File "lib/python2.7/site-packages/google/cloud/bigquery/_helpers.py", line 61, in _record_from_json
    for subfield, cell in zip(field.fields, value['f']):
KeyError: 'f'
```

---

Stepping into the debugger:

``` python
ipdb> value
{u'v': {u'f': [{u'v': u'hamlet'}, {u'v': u'5318'}]}}
ipdb> field.__dict__
{'field_type': u'RECORD', 'description': None, 'name': u'title', 'fields': [<google.cloud.bigquery.schema.SchemaField object at 0x7f4ba9227550>, <google.cloud.bigquery.schema.SchemaField object at 0x7f4ba9227690>], 'mode': u'REPEATED'}
```

(Added by @dhermes)



Discussion:
A simple statement works fine

``` python
>>> from google.cloud import bigquery
>>> client = bigquery.Client()
>>> query = client.run_sync_query('SELECT 1')
>>> query.use_legacy_sql = False
>>> query.run()
>>> query.rows
[(1,)]
```

Sending a query with nested types also works fine

``` python
In [1]: from google.cloud import bigquery

In [2]: client = bigquery.Client()

In [3]: sql_str = (
   ...:     'SELECT ARRAY['
   ...:     'STRUCT("zx83" AS id, 1 AS a, 2 AS b), '
   ...:     '("f8f7", 4, 7)'
   ...:     '] '
   ...:     'd')

In [4]: query = client.run_sync_query(sql_str)

In [5]: query.use_legacy_sql = False

In [6]: query.run()

In [7]: query.__dict__
Out[7]:
{'_client': <google.cloud.bigquery.client.Client at 0x7fb7cf39a290>,
 '_configuration': <google.cloud.bigquery.query._SyncQueryConfiguration at 0x7fb7cf3b0fd0>,
 '_job': None,
 '_properties': {u'cacheHit': False,
  u'jobComplete': True,
  u'jobReference': {u'jobId': u'job_ruoSHLwjeg_lSfYZq8dmrjQ0Eg4',
   u'projectId': u'precise-truck-742'},
  u'kind': u'bigquery#queryResponse',
  u'rows': [{u'f': [{u'v': [{u'v': {u'f': [{u'v': u'zx83'},
          {u'v': u'1'},
          {u'v': u'2'}]}},
       {u'v': {u'f': [{u'v': u'f8f7'}, {u'v': u'4'}, {u'v': u'7'}]}}]}]}],
  u'schema': {u'fields': [{u'fields': [{u'mode': u'NULLABLE',
       u'name': u'id',
       u'type': u'STRING'},
      {u'mode': u'NULLABLE', u'name': u'a', u'type': u'INTEGER'},
      {u'mode': u'NULLABLE', u'name': u'b', u'type': u'INTEGER'}],
     u'mode': u'REPEATED',
     u'name': u'd',
     u'type': u'RECORD'}]},
  u'totalBytesProcessed': u'0',
  u'totalRows': u'1'},
 '_udf_resources': (),
 'query': 'SELECT ARRAY[STRUCT("zx83" AS id, 1 AS a, 2 AS b), ("f8f7", 4, 7)] d'}
```

but the `rows` accessor fails

``` python
In [8]: query.rows
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-8-27b591eefa89> in <module>()
----> 1 query.rows

.../google/cloud/bigquery/query.pyc in rows(self)
    223         :returns: fields describing the schema (None until set by the server).
    224         """
--> 225         return _rows_from_json(self._properties.get('rows', ()), self.schema)
    226
    227     @property

.../google/cloud/bigquery/_helpers.pyc in _rows_from_json(rows, schema)
     96             if field.mode == 'REPEATED':
     97                 row_data.append([converter(item, field)
---> 98                                  for item in cell['v']])
     99             else:
    100                 row_data.append(converter(cell['v'], field))

.../google/cloud/bigquery/_helpers.pyc in _record_from_json(value, field)
     59     if _not_null(value, field):
     60         record = {}
---> 61         for subfield, cell in zip(field.fields, value['f']):
     62             converter = _CELLDATA_FROM_JSON[subfield.field_type]
     63             if field.mode == 'REPEATED':

KeyError: 'f'
```

Just for fun, here are those last properties as JSON:

``` json
{
  "cacheHit": false, 
  "jobComplete": true, 
  "jobReference": {
    "jobId": "job_ruoSHLwjeg_lSfYZq8dmrjQ0Eg4", 
    "projectId": "precise-truck-742"
  }, 
  "kind": "bigquery#queryResponse", 
  "rows": [
    {
      "f": [
        {
          "v": [
            {
              "v": {
                "f": [
                  {
                    "v": "zx83"
                  }, 
                  {
                    "v": "1"
                  }, 
                  {
                    "v": "2"
                  }
                ]
              }
            }, 
            {
              "v": {
                "f": [
                  {
                    "v": "f8f7"
                  }, 
                  {
                    "v": "4"
                  }, 
                  {
                    "v": "7"
                  }
                ]
              }
            }
          ]
        }
      ]
    }
  ], 
  "schema": {
    "fields": [
      {
        "fields": [
          {
            "mode": "NULLABLE", 
            "name": "id", 
            "type": "STRING"
          }, 
          {
            "mode": "NULLABLE", 
            "name": "a", 
            "type": "INTEGER"
          }, 
          {
            "mode": "NULLABLE", 
            "name": "b", 
            "type": "INTEGER"
          }
        ], 
        "mode": "REPEATED", 
        "name": "d", 
        "type": "RECORD"
      }
    ]
  }, 
  "totalBytesProcessed": "0", 
  "totalRows": "1"
}
```

and where can I get alias names? in a query like below I am expecting to access results in a dict way like `result.a` `result.b` or `result["a"]` `result["b"]` because in my actual app (in nodejs) I have more complicated query generated schema, there are more fields, to access all those fields in the tuple `(1, 2)` way is not convenient and error prone; please let me know should it be another issue by its own? or can be documentation request if you already have it (but I don't see an example in current doc)

https://googlecloudplatform.github.io/google-cloud-python/stable/bigquery-query.html

``` py
In [56]: query = client.run_sync_query('SELECT 1 AS a, 2 AS b'); query.use_legacy_sql = False; query.run()

In [57]: query.__dict__
Out[57]: 
{'_client': <gcloud.bigquery.client.Client at 0x7f89a047ad90>,
 '_configuration': <gcloud.bigquery.query._SyncQueryConfiguration at 0x7f89a0404710>,
 '_job': None,
 '_properties': {u'cacheHit': False,
  u'jobComplete': True,
  u'jobReference': {u'jobId': u'job_POdEXqw6MgHtL-WDZX21BA7_fII',
   u'projectId': u'hb-inspector'},
  u'kind': u'bigquery#queryResponse',
  u'rows': [{u'f': [{u'v': u'1'}, {u'v': u'2'}]}],
  u'schema': {u'fields': [{u'mode': u'NULLABLE',
     u'name': u'a',
     u'type': u'INTEGER'},
    {u'mode': u'NULLABLE', u'name': u'b', u'type': u'INTEGER'}]},
  u'totalBytesProcessed': u'0',
  u'totalRows': u'1'},
 '_udf_resources': (),
 'query': 'SELECT 1 AS a, 2 AS b'}

In [58]: query.rows
Out[58]: [(1, 2)]
```

I just found bq tool has a command switch --apilog can save the detail log, exposed the library used by bq tool's python code is `google-api-python-client`; that one is able to handle nested google-api-python-client; at least not crashing, but the data types handling are still not there google/google-api-python-client#281

INFO:root:user-agent: google-api-python-client/1.3.1 (gzip)

Bump - this is blocking a critical sample.

@tseaver You still planning to handle?

Another bump here.

**blocked on this** can't use google-cloud-python with the new BigQuery standard SQL

Another bump, this is blocking samples and usage of this service.

/cc @omaray @jgeewax 

/cc @stephenplusplus 
Another bump, blocked on this.

Currently using a **manual** fix to:
`google/cloud/bigquery/_helpers.py, line 61, in _record_from_json`
Changed from
```python
 for subfield, cell in zip(field.fields, value['f']):
```
to
```python
record_iter = None
if 'f' in value:
    record_iter = zip(field.fields, value['f'])
elif 'v' in value and 'f' in value['v']:
    record_iter = zip(field.fields, value['v']['f'])
if record_iter:
    for subfield, cell in record_iter:
```

Also changed `line 63`, it's referencing the parent field mode instead of the subfield.
Changed from
```python
if field.mode == 'REPEATED':
```
to
```python
if subfield.mode == 'REPEATED':
```

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -58,10 +58,11 @@ def _record_from_json(value, field):
     """Coerce 'value' to a mapping, if set or not nullable."""
     if _not_null(value, field):
         record = {}
-        for subfield, cell in zip(field.fields, value['f']):
+        record_iter = zip(field.fields, value['f'])
+        for subfield, cell in record_iter:
             converter = _CELLDATA_FROM_JSON[subfield.field_type]
-            if field.mode == 'REPEATED':
-                value = [converter(item, subfield) for item in cell['v']]
+            if subfield.mode == 'REPEATED':
+                value = [converter(item['v'], subfield) for item in cell['v']]
             else:
                 value = converter(cell['v'], subfield)
             record[subfield.name] = value
@@ -103,7 +104,7 @@ def _row_from_json(row, schema):
     for field, cell in zip(schema, row['f']):
         converter = _CELLDATA_FROM_JSON[field.field_type]
         if field.mode == 'REPEATED':
-            row_data.append([converter(item, field)
+            row_data.append([converter(item['v'], field)
                              for item in cell['v']])
         else:
             row_data.append(converter(cell['v'], field))


Test patch:
diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py
--- a/bigquery/unit_tests/test__helpers.py
+++ b/bigquery/unit_tests/test__helpers.py
@@ -186,7 +186,7 @@ def test_w_scalar_subfield(self):
     def test_w_repeated_subfield(self):
         subfield = _Field('REPEATED', 'color', 'STRING')
         field = _Field('REQUIRED', fields=[subfield])
-        value = {'f': [{'v': ['red', 'yellow', 'blue']}]}
+        value = {'f': [{'v': [{'v': 'red'}, {'v': 'yellow'}, {'v': 'blue'}]}]}
         coerced = self._call_fut(value, field)
         self.assertEqual(coerced, {'color': ['red', 'yellow', 'blue']})
 
@@ -234,6 +234,97 @@ def test_w_string_value(self):
         self.assertEqual(coerced, 'Wonderful!')
 
 
+class Test_row_from_json(unittest.TestCase):
+
+    def _call_fut(self, row, schema):
+        from google.cloud.bigquery._helpers import _row_from_json
+        return _row_from_json(row, schema)
+
+    def test_w_single_scalar_column(self):
+        # SELECT 1 AS col
+        col = _Field('REQUIRED', 'col', 'INTEGER')
+        row = {u'f': [{u'v': u'1'}]}
+        self.assertEqual(self._call_fut(row, schema=[col]), (1,))
+
+    def test_w_single_struct_column(self):
+        # SELECT (1, 2) AS col
+        sub_1 = _Field('REQUIRED', 'sub_1', 'INTEGER')
+        sub_2 = _Field('REQUIRED', 'sub_2', 'INTEGER')
+        col = _Field('REQUIRED', 'col', 'RECORD', fields=[sub_1, sub_2])
+        row = {u'f': [{u'v': {u'f': [{u'v': u'1'}, {u'v': u'2'}]}}]}
+        self.assertEqual(self._call_fut(row, schema=[col]),
+                         ({'sub_1': 1, 'sub_2': 2},))
+
+    def test_w_single_array_column(self):
+        # SELECT [1, 2, 3] as col
+        col = _Field('REPEATED', 'col', 'INTEGER')
+        row = {u'f': [{u'v': [{u'v': u'1'}, {u'v': u'2'}, {u'v': u'3'}]}]}
+        self.assertEqual(self._call_fut(row, schema=[col]),
+                         ([1, 2, 3],))
+
+    def test_w_struct_w_nested_array_column(self):
+        # SELECT ([1, 2], 3, [4, 5]) as col
+        first = _Field('REPEATED', 'first', 'INTEGER')
+        second = _Field('REQUIRED', 'second', 'INTEGER')
+        third = _Field('REPEATED', 'third', 'INTEGER')
+        col = _Field('REQUIRED', 'col', 'RECORD',
+                     fields=[first, second, third])
+        row = {
+            u'f': [
+                {u'v': {
+                    u'f': [
+                        {u'v': [{u'v': u'1'}, {u'v': u'2'}]},
+                        {u'v': u'3'},
+                        {u'v': [{u'v': u'4'}, {u'v': u'5'}]}
+                    ]
+                }},
+            ]
+        }
+        self.assertEqual(
+            self._call_fut(row, schema=[col]),
+            ({u'first': [1, 2], u'second': 3, u'third': [4, 5]},))
+
+    def test_w_array_of_struct(self):
+        # SELECT [(1, 2, 3), (4, 5, 6)] as col
+        first = _Field('REQUIRED', 'first', 'INTEGER')
+        second = _Field('REQUIRED', 'second', 'INTEGER')
+        third = _Field('REQUIRED', 'third', 'INTEGER')
+        col = _Field('REPEATED', 'col', 'RECORD',
+                     fields=[first, second, third])
+        row = {u'f': [{u'v': [
+            {u'v': {u'f': [{u'v': u'1'}, {u'v': u'2'}, {u'v': u'3'}]}},
+            {u'v': {u'f': [{u'v': u'4'}, {u'v': u'5'}, {u'v': u'6'}]}},
+        ]}]}
+        self.assertEqual(
+            self._call_fut(row, schema=[col]),
+            ([
+                {u'first': 1, u'second': 2, u'third': 3},
+                {u'first': 4, u'second': 5, u'third': 6},
+            ],))
+
+    def test_w_array_of_struct_w_array(self):
+        # SELECT [([1, 2, 3], 4), ([5, 6], 7)]
+        first = _Field('REPEATED', 'first', 'INTEGER')
+        second = _Field('REQUIRED', 'second', 'INTEGER')
+        col = _Field('REPEATED', 'col', 'RECORD', fields=[first, second])
+        row = {u'f': [{u'v': [
+            {u'v': {u'f': [
+                {u'v': [{u'v': u'1'}, {u'v': u'2'}, {u'v': u'3'}]},
+                {u'v': u'4'}
+            ]}},
+            {u'v': {u'f': [
+                {u'v': [{u'v': u'5'}, {u'v': u'6'}]},
+                {u'v': u'7'}
+            ]}}
+        ]}]}
+        self.assertEqual(
+            self._call_fut(row, schema=[col]),
+            ([
+                {u'first': [1, 2, 3], u'second': 4},
+                {u'first': [5, 6], u'second': 7},
+            ],))
+
+
 class Test_rows_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -253,12 +344,12 @@ def test_w_record_subfield(self):
             {'f': [
                 {'v': 'Phred Phlyntstone'},
                 {'v': {'f': [{'v': '800'}, {'v': '555-1212'}, {'v': 1}]}},
-                {'v': ['orange', 'black']},
+                {'v': [{'v': 'orange'}, {'v': 'black'}]},
             ]},
             {'f': [
                 {'v': 'Bharney Rhubble'},
                 {'v': {'f': [{'v': '877'}, {'v': '768-5309'}, {'v': 2}]}},
-                {'v': ['brown']},
+                {'v': [{'v': 'brown'}]},
             ]},
             {'f': [
                 {'v': 'Wylma Phlyntstone'},
diff --git a/bigquery/unit_tests/test_table.py b/bigquery/unit_tests/test_table.py
--- a/bigquery/unit_tests/test_table.py
+++ b/bigquery/unit_tests/test_table.py
@@ -1173,22 +1173,27 @@ def test_fetch_data_w_repeated_fields(self):
             'pageToken': TOKEN,
             'rows': [
                 {'f': [
-                    {'v': ['red', 'green']},
-                    {'v': [{'f': [{'v': ['1', '2']},
-                                  {'v': ['3.1415', '1.414']}]}]},
+                    {'v': [{'v': 'red'}, {'v': 'green'}]},
+                    {'v': [{
+                        'v': {
+                            'f': [
+                                {'v': [{'v': '1'}, {'v': '2'}]},
+                                {'v': [{'v': '3.1415'}, {'v': '1.414'}]},
+                            ]}
+                    }]},
                 ]},
             ]
         }
         conn = _Connection(DATA)
         client = _Client(project=self.PROJECT, connection=conn)
         dataset = _Dataset(client)
-        full_name = SchemaField('color', 'STRING', mode='REPEATED')
+        color = SchemaField('color', 'STRING', mode='REPEATED')
         index = SchemaField('index', 'INTEGER', 'REPEATED')
         score = SchemaField('score', 'FLOAT', 'REPEATED')
         struct = SchemaField('struct', 'RECORD', mode='REPEATED',
                              fields=[index, score])
         table = self._make_one(self.TABLE_NAME, dataset=dataset,
-                               schema=[full_name, struct])
+                               schema=[color, struct])
 
         iterator = table.fetch_data()
         page = six.next(iterator.pages)
diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py
--- a/system_tests/bigquery.py
+++ b/system_tests/bigquery.py
@@ -478,3 +478,49 @@ def _job_done(instance):
         # them here.  The best we can do is not that the API call didn't
         # raise an error, and that the job completed (in the `retry()`
         # above).
+
+    def test_sync_query_w_nested_arrays_and_structs(self):
+        EXAMPLES = [
+            {
+                'sql': 'SELECT 1',
+                'expected': 1,
+            },
+            {
+                'sql': 'SELECT (1, 2)',
+                'expected': {'_field_1': 1, '_field_2': 2},
+            },
+            {
+                'sql': 'SELECT [1, 2, 3]',
+                'expected': [1, 2, 3],
+            },
+            {
+                'sql': 'SELECT ([1, 2], 3, [4, 5])',
+                'expected':
+                    {'_field_1': [1, 2], '_field_2': 3, '_field_3': [4, 5]},
+            },
+            {
+                'sql': 'SELECT [(1, 2, 3), (4, 5, 6)]',
+                'expected': [
+                    {'_field_1': 1, '_field_2': 2, '_field_3': 3},
+                    {'_field_1': 4, '_field_2': 5, '_field_3': 6},
+                ],
+            },
+            {
+                'sql': 'SELECT [([1, 2, 3], 4), ([5, 6], 7)]',
+                'expected': [
+                    {u'_field_1': [1, 2, 3], u'_field_2': 4},
+                    {u'_field_1': [5, 6], u'_field_2': 7},
+                ],
+            },
+            {
+                'sql': 'SELECT ARRAY(SELECT STRUCT([1, 2]))',
+                'expected': [{u'_field_1': [1, 2]}],
+            },
+        ]
+        for example in EXAMPLES:
+            query = Config.CLIENT.run_sync_query(example['sql'])
+            query.use_legacy_sql = False
+            query.run()
+            self.assertEqual(len(query.rows), 1)
+            self.assertEqual(len(query.rows[0]), 1)
+            self.assertEqual(query.rows[0][0], example['expected'])