Repository: googleapis/google-cloud-python

Issue:
bigquery.ArrayQueryParameter should be able to accept structs as elements of the array
I [commented](https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2901#discussion_r94066868) that setting the types on array query parameters needs to be recursive since the array could contain structs. Note that [arrays cannot contain arrays](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type) (the workaround is to contain a struct which can then contain an array).

It's not clear how one would do this with the current `ArrayQueryParameter` constructor.

```
bigquery.ArrayQueryParameter(
    name='myarray', 
    type='STRUCT',  # Note: this isn't enough information to construct the type object.
    values=?  # What would go here? Dictionaries? That doesn't match other uses of StructQueryParameter.
)
```

Maybe we need to change how array query parameters are constructed? Maybe it should look more like the StructQueryParameter constructor?

```
bigquery.ArrayQueryParameter(name, *sub_params)
```

It'd be a pain for scalar values and the names would have to be ignored on the `sub_params` if provided, but it's more flexible.

*An aside*

*I'd actually love if we could get rid of all the `XQueryParameter` classes and use lists, dicts, and ints, floats, etc. But I see that it might be ambiguous for [some types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types) especially `DATETIME` (no time zone) vs `TIMESTAMP` (stored as UTC time).*


Discussion:

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -500,18 +500,23 @@ def to_api_repr(self):
         :returns: JSON mapping
         """
         values = self.values
-        converter = _SCALAR_VALUE_TO_JSON.get(self.array_type)
-        if converter is not None:
-            values = [converter(value) for value in values]
+        if self.array_type == 'RECORD':
+            reprs = [value.to_api_repr() for value in values]
+            a_type = reprs[0]['parameterType']
+            a_values = [repr_['parameterValue'] for repr_ in reprs]
+        else:
+            a_type = {'type': self.array_type}
+            converter = _SCALAR_VALUE_TO_JSON.get(self.array_type)
+            if converter is not None:
+                values = [converter(value) for value in values]
+            a_values = [{'value': value} for value in values]
         resource = {
             'parameterType': {
                 'type': 'ARRAY',
-                'arrayType': {
-                    'type': self.array_type,
-                },
+                'arrayType': a_type,
             },
             'parameterValue': {
-                'arrayValues': [{'value': value} for value in values],
+                'arrayValues': a_values,
             },
         }
         if self.name is not None:
@@ -531,9 +536,18 @@ class StructQueryParameter(AbstractQueryParameter):
     """
     def __init__(self, name, *sub_params):
         self.name = name
-        self.struct_types = OrderedDict(
-            (sub.name, sub.type_) for sub in sub_params)
-        self.struct_values = {sub.name: sub.value for sub in sub_params}
+        types = self.struct_types = OrderedDict()
+        values = self.struct_values = {}
+        for sub in sub_params:
+            if isinstance(sub, self.__class__):
+                types[sub.name] = 'STRUCT'
+                values[sub.name] = sub
+            elif isinstance(sub, ArrayQueryParameter):
+                types[sub.name] = 'ARRAY'
+                values[sub.name] = sub
+            else:
+                types[sub.name] = sub.type_
+                values[sub.name] = sub.value
 
     @classmethod
     def positional(cls, *sub_params):
@@ -576,21 +590,25 @@ def to_api_repr(self):
         :rtype: dict
         :returns: JSON mapping
         """
-        types = [
-            {'name': key, 'type': {'type': value}}
-            for key, value in self.struct_types.items()
-        ]
+        s_types = {}
         values = {}
         for name, value in self.struct_values.items():
-            converter = _SCALAR_VALUE_TO_JSON.get(self.struct_types[name])
-            if converter is not None:
-                value = converter(value)
-            values[name] = {'value': value}
+            type_ = self.struct_types[name]
+            if type_ in ('STRUCT', 'ARRAY'):
+                repr_ = value.to_api_repr()
+                s_types[name] = {'name': name, 'type': repr_['parameterType']}
+                values[name] = repr_['parameterValue']
+            else:
+                s_types[name] = {'name': name, 'type': {'type': type_}}
+                converter = _SCALAR_VALUE_TO_JSON.get(type_)
+                if converter is not None:
+                    value = converter(value)
+                values[name] = {'value': value}
 
         resource = {
             'parameterType': {
                 'type': 'STRUCT',
-                'structTypes': types,
+                'structTypes': [s_types[key] for key in self.struct_types],
             },
             'parameterValue': {
                 'structValues': values,


Test patch:
diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py
--- a/bigquery/unit_tests/test__helpers.py
+++ b/bigquery/unit_tests/test__helpers.py
@@ -1089,6 +1089,12 @@ def test_to_api_repr_w_unknown_type(self):
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
 
+def _make_subparam(name, type_, value):
+    from google.cloud.bigquery._helpers import ScalarQueryParameter
+
+    return ScalarQueryParameter(name, type_, value)
+
+
 class Test_ArrayQueryParameter(unittest.TestCase):
 
     @staticmethod
@@ -1230,6 +1236,36 @@ def test_to_api_repr_w_unknown_type(self):
         param = klass.positional(array_type='UNKNOWN', values=['unknown'])
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test_to_api_repr_w_record_type(self):
+        from google.cloud.bigquery._helpers import StructQueryParameter
+
+        EXPECTED = {
+            'parameterType': {
+                'type': 'ARRAY',
+                'arrayType': {
+                    'type': 'STRUCT',
+                    'structTypes': [
+                        {'name': 'foo', 'type': {'type': 'STRING'}},
+                        {'name': 'bar', 'type': {'type': 'INT64'}},
+                    ],
+                },
+            },
+            'parameterValue': {
+                'arrayValues': [{
+                    'structValues': {
+                        'foo': {'value': 'Foo'},
+                        'bar': {'value': '123'},
+                    }
+                }]
+            },
+        }
+        one = _make_subparam('foo', 'STRING', 'Foo')
+        another = _make_subparam('bar', 'INT64', 123)
+        struct = StructQueryParameter.positional(one, another)
+        klass = self._get_target_class()
+        param = klass.positional(array_type='RECORD', values=[struct])
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
 
 class Test_StructQueryParameter(unittest.TestCase):
 
@@ -1242,23 +1278,17 @@ def _get_target_class():
     def _make_one(self, *args, **kw):
         return self._get_target_class()(*args, **kw)
 
-    @staticmethod
-    def _make_subparam(name, type_, value):
-        from google.cloud.bigquery._helpers import ScalarQueryParameter
-
-        return ScalarQueryParameter(name, type_, value)
-
     def test_ctor(self):
-        sub_1 = self._make_subparam('bar', 'INT64', 123)
-        sub_2 = self._make_subparam('baz', 'STRING', 'abc')
+        sub_1 = _make_subparam('bar', 'INT64', 123)
+        sub_2 = _make_subparam('baz', 'STRING', 'abc')
         param = self._make_one('foo', sub_1, sub_2)
         self.assertEqual(param.name, 'foo')
         self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'})
         self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'})
 
     def test_positional(self):
-        sub_1 = self._make_subparam('bar', 'INT64', 123)
-        sub_2 = self._make_subparam('baz', 'STRING', 'abc')
+        sub_1 = _make_subparam('bar', 'INT64', 123)
+        sub_2 = _make_subparam('baz', 'STRING', 'abc')
         klass = self._get_target_class()
         param = klass.positional(sub_1, sub_2)
         self.assertEqual(param.name, None)
@@ -1327,8 +1357,8 @@ def test_to_api_repr_w_name(self):
                 },
             },
         }
-        sub_1 = self._make_subparam('bar', 'INT64', 123)
-        sub_2 = self._make_subparam('baz', 'STRING', 'abc')
+        sub_1 = _make_subparam('bar', 'INT64', 123)
+        sub_2 = _make_subparam('baz', 'STRING', 'abc')
         param = self._make_one('foo', sub_1, sub_2)
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
@@ -1348,12 +1378,75 @@ def test_to_api_repr_wo_name(self):
                 },
             },
         }
-        sub_1 = self._make_subparam('bar', 'INT64', 123)
-        sub_2 = self._make_subparam('baz', 'STRING', 'abc')
+        sub_1 = _make_subparam('bar', 'INT64', 123)
+        sub_2 = _make_subparam('baz', 'STRING', 'abc')
         klass = self._get_target_class()
         param = klass.positional(sub_1, sub_2)
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test_to_api_repr_w_nested_array(self):
+        from google.cloud.bigquery._helpers import ArrayQueryParameter
+
+        EXPECTED = {
+            'name': 'foo',
+            'parameterType': {
+                'type': 'STRUCT',
+                'structTypes': [
+                    {'name': 'bar', 'type': {'type': 'STRING'}},
+                    {'name': 'baz', 'type': {
+                        'type': 'ARRAY',
+                        'arrayType': {'type': 'INT64'},
+                    }},
+                ],
+            },
+            'parameterValue': {
+                'structValues': {
+                    'bar': {'value': 'abc'},
+                    'baz': {'arrayValues': [
+                        {'value': '123'},
+                        {'value': '456'},
+                    ]},
+                },
+            },
+        }
+        scalar = _make_subparam('bar', 'STRING', 'abc')
+        array = ArrayQueryParameter('baz', 'INT64', [123, 456])
+        param = self._make_one('foo', scalar, array)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
+    def test_to_api_repr_w_nested_struct(self):
+        EXPECTED = {
+            'name': 'foo',
+            'parameterType': {
+                'type': 'STRUCT',
+                'structTypes': [
+                    {'name': 'bar', 'type': {'type': 'STRING'}},
+                    {'name': 'baz', 'type': {
+                        'type': 'STRUCT',
+                        'structTypes': [
+                            {'name': 'qux', 'type': {'type': 'INT64'}},
+                            {'name': 'spam', 'type': {'type': 'BOOL'}},
+                        ],
+                    }},
+                ],
+            },
+            'parameterValue': {
+                'structValues': {
+                    'bar': {'value': 'abc'},
+                    'baz': {'structValues': {
+                        'qux': {'value': '123'},
+                        'spam': {'value': 'true'},
+                    }},
+                },
+            },
+        }
+        scalar_1 = _make_subparam('bar', 'STRING', 'abc')
+        scalar_2 = _make_subparam('qux', 'INT64', 123)
+        scalar_3 = _make_subparam('spam', 'BOOL', True)
+        sub = self._make_one('baz', scalar_2, scalar_3)
+        param = self._make_one('foo', scalar_1, sub)
+        self.assertEqual(param.to_api_repr(), EXPECTED)
+
 
 class Test_QueryParametersProperty(unittest.TestCase):
 
diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py
--- a/system_tests/bigquery.py
+++ b/system_tests/bigquery.py
@@ -532,39 +532,9 @@ def test_sync_query_w_legacy_sql_types(self):
             self.assertEqual(query.rows[0][0], example['expected'])
 
     def test_sync_query_w_standard_sql_types(self):
-        from google.cloud.bigquery._helpers import ArrayQueryParameter
-        from google.cloud.bigquery._helpers import ScalarQueryParameter
-        from google.cloud.bigquery._helpers import StructQueryParameter
-        question = 'What is the answer to life, the universe, and everything?'
-        question_param = ScalarQueryParameter(
-            name='question', type_='STRING', value=question)
-        answer = 42
-        answer_param = ScalarQueryParameter(
-            name='answer', type_='INT64', value=answer)
-        pi = 3.1415926
-        pi_param = ScalarQueryParameter(
-            name='pi', type_='FLOAT64', value=pi)
-        truthy = True
-        truthy_param = ScalarQueryParameter(
-            name='truthy', type_='BOOL', value=truthy)
-        beef = b'DEADBEEF'
-        beef_param = ScalarQueryParameter(
-            name='beef', type_='BYTES', value=beef)
         naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
         stamp = '%s %s' % (naive.date().isoformat(), naive.time().isoformat())
-        naive_param = ScalarQueryParameter(
-            name='naive', type_='DATETIME', value=naive)
-        naive_date_param = ScalarQueryParameter(
-            name='naive_date', type_='DATE', value=naive.date())
-        naive_time_param = ScalarQueryParameter(
-            name='naive_time', type_='TIME', value=naive.time())
         zoned = naive.replace(tzinfo=UTC)
-        zoned_param = ScalarQueryParameter(
-            name='zoned', type_='TIMESTAMP', value=zoned)
-        array_param = ArrayQueryParameter(
-            name='array_param', array_type='INT64', values=[1, 2])
-        struct_param = StructQueryParameter(
-            'hitchhiker', question_param, answer_param)
         EXAMPLES = [
             {
                 'sql': 'SELECT 1',
@@ -641,6 +611,78 @@ def test_sync_query_w_standard_sql_types(self):
                 'sql': 'SELECT ARRAY(SELECT STRUCT([1, 2]))',
                 'expected': [{u'_field_1': [1, 2]}],
             },
+        ]
+        for example in EXAMPLES:
+            query = Config.CLIENT.run_sync_query(example['sql'])
+            query.use_legacy_sql = False
+            query.run()
+            self.assertEqual(len(query.rows), 1)
+            self.assertEqual(len(query.rows[0]), 1)
+            self.assertEqual(query.rows[0][0], example['expected'])
+
+    def test_sync_query_w_query_params(self):
+        from google.cloud.bigquery._helpers import ArrayQueryParameter
+        from google.cloud.bigquery._helpers import ScalarQueryParameter
+        from google.cloud.bigquery._helpers import StructQueryParameter
+        question = 'What is the answer to life, the universe, and everything?'
+        question_param = ScalarQueryParameter(
+            name='question', type_='STRING', value=question)
+        answer = 42
+        answer_param = ScalarQueryParameter(
+            name='answer', type_='INT64', value=answer)
+        pi = 3.1415926
+        pi_param = ScalarQueryParameter(
+            name='pi', type_='FLOAT64', value=pi)
+        truthy = True
+        truthy_param = ScalarQueryParameter(
+            name='truthy', type_='BOOL', value=truthy)
+        beef = b'DEADBEEF'
+        beef_param = ScalarQueryParameter(
+            name='beef', type_='BYTES', value=beef)
+        naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
+        naive_param = ScalarQueryParameter(
+            name='naive', type_='DATETIME', value=naive)
+        naive_date_param = ScalarQueryParameter(
+            name='naive_date', type_='DATE', value=naive.date())
+        naive_time_param = ScalarQueryParameter(
+            name='naive_time', type_='TIME', value=naive.time())
+        zoned = naive.replace(tzinfo=UTC)
+        zoned_param = ScalarQueryParameter(
+            name='zoned', type_='TIMESTAMP', value=zoned)
+        array_param = ArrayQueryParameter(
+            name='array_param', array_type='INT64', values=[1, 2])
+        struct_param = StructQueryParameter(
+            'hitchhiker', question_param, answer_param)
+        phred_name = 'Phred Phlyntstone'
+        phred_name_param = ScalarQueryParameter(
+            name='name', type_='STRING', value=phred_name)
+        phred_age = 32
+        phred_age_param = ScalarQueryParameter(
+            name='age', type_='INT64', value=phred_age)
+        phred_param = StructQueryParameter(
+            None, phred_name_param, phred_age_param)
+        bharney_name = 'Bharney Rhubbyl'
+        bharney_name_param = ScalarQueryParameter(
+            name='name', type_='STRING', value=bharney_name)
+        bharney_age = 31
+        bharney_age_param = ScalarQueryParameter(
+            name='age', type_='INT64', value=bharney_age)
+        bharney_param = StructQueryParameter(
+            None, bharney_name_param, bharney_age_param)
+        characters_param = ArrayQueryParameter(
+            name=None, array_type='RECORD',
+            values=[phred_param, bharney_param])
+        hero_param = StructQueryParameter(
+            'hero', phred_name_param, phred_age_param)
+        sidekick_param = StructQueryParameter(
+            'sidekick', bharney_name_param, bharney_age_param)
+        roles_param = StructQueryParameter(
+            'roles', hero_param, sidekick_param)
+        friends_param = ArrayQueryParameter(
+            name='friends', array_type='STRING',
+            values=[phred_name, bharney_name])
+        with_friends_param = StructQueryParameter(None, friends_param)
+        EXAMPLES = [
             {
                 'sql': 'SELECT @question',
                 'expected': question,
@@ -696,11 +738,34 @@ def test_sync_query_w_standard_sql_types(self):
                 'expected': ({'_field_1': question, '_field_2': answer}),
                 'query_parameters': [struct_param],
             },
+            {
+                'sql': 'SELECT ?',
+                'expected': [
+                    {'name': phred_name, 'age': phred_age},
+                    {'name': bharney_name, 'age': bharney_age},
+                ],
+                'query_parameters': [characters_param],
+            },
+            {
+                'sql': 'SELECT @roles',
+                'expected': {
+                    'hero': {'name': phred_name, 'age': phred_age},
+                    'sidekick': {'name': bharney_name, 'age': bharney_age},
+                },
+                'query_parameters': [roles_param],
+            },
+            {
+                'sql': 'SELECT ?',
+                'expected': {
+                    'friends': [phred_name, bharney_name],
+                },
+                'query_parameters': [with_friends_param],
+            },
         ]
         for example in EXAMPLES:
             query = Config.CLIENT.run_sync_query(
                 example['sql'],
-                query_parameters=example.get('query_parameters', ()))
+                query_parameters=example['query_parameters'])
             query.use_legacy_sql = False
             query.run()
             self.assertEqual(len(query.rows), 1)