Repository: googleapis/google-cloud-python

Issue:
BigQuery: ArrayQueryParameter.from_api_repr fails with STRUCT query parameter values
I encountered a system test error on the `bigquery-b2` work when I modified some system tests to use QueryJob to pass in query parameters. The job resource that comes back cannot be parsed because of the query parameters in the query statistics.

The resource which is show the problem is this one:

        RESOURCE = {
            'parameterType': {
                'type': 'ARRAY',
                'arrayType': {
                    'type': 'STRUCT',
                    'structTypes': [
                        {
                            'name': 'name',
                            'type': {'type': 'STRING'}},
                        ,
                        {
                            'name': 'age',
                            'type': {'type': 'INT64'},
                        },
                    ],
                },
            },
            'parameterValue': {
                'arrayValues': [
                    {
                        'structValues': {
                            'name': {'value': 'Phred Phlyntstone'},
                            'age': {'value': '32'},
                        },
                    },
                    {
                        'structValues': {
                            'name': {
                                'value': 'Bharney Rhubbyl',
                            },
                            'age': {'value': '31'},
                        },
                    },
                ],
            },
        }

I'm currently investigating a fix. Will make a PR to `master` rather than `bigquery-b2` when ready because this issue will affect that branch too and I want a fix out possibly before Beta 2 is ready.


Discussion:

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py
--- a/bigquery/google/cloud/bigquery/_helpers.py
+++ b/bigquery/google/cloud/bigquery/_helpers.py
@@ -16,6 +16,7 @@
 
 import base64
 from collections import OrderedDict
+import copy
 import datetime
 
 from google.cloud._helpers import UTC
@@ -468,6 +469,31 @@ def to_api_repr(self):
             resource['name'] = self.name
         return resource
 
+    def _key(self):
+        """A tuple key that uniquely describes this field.
+
+        Used to compute this instance's hashcode and evaluate equality.
+
+        Returns:
+            tuple: The contents of this :class:`ScalarQueryParameter`.
+        """
+        return (
+            self.name,
+            self.type_.upper(),
+            self.value,
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, ScalarQueryParameter):
+            return NotImplemented
+        return self._key() == other._key()
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __repr__(self):
+        return 'ScalarQueryParameter{}'.format(self._key())
+
 
 class ArrayQueryParameter(AbstractQueryParameter):
     """Named / positional query parameters for array values.
@@ -507,15 +533,24 @@ def positional(cls, array_type, values):
         return cls(None, array_type, values)
 
     @classmethod
-    def from_api_repr(cls, resource):
-        """Factory: construct parameter from JSON resource.
-
-        :type resource: dict
-        :param resource: JSON mapping of parameter
+    def _from_api_repr_struct(cls, resource):
+        name = resource.get('name')
+        converted = []
+        # We need to flatten the array to use the StructQueryParameter
+        # parse code.
+        resource_template = {
+            # The arrayType includes all the types of the fields of the STRUCT
+            'parameterType': resource['parameterType']['arrayType']
+        }
+        for array_value in resource['parameterValue']['arrayValues']:
+            struct_resource = copy.deepcopy(resource_template)
+            struct_resource['parameterValue'] = array_value
+            struct_value = StructQueryParameter.from_api_repr(struct_resource)
+            converted.append(struct_value)
+        return cls(name, 'STRUCT', converted)
 
-        :rtype: :class:`ArrayQueryParameter`
-        :returns: instance
-        """
+    @classmethod
+    def _from_api_repr_scalar(cls, resource):
         name = resource.get('name')
         array_type = resource['parameterType']['arrayType']['type']
         values = [
@@ -526,6 +561,21 @@ def from_api_repr(cls, resource):
             _CELLDATA_FROM_JSON[array_type](value, None) for value in values]
         return cls(name, array_type, converted)
 
+    @classmethod
+    def from_api_repr(cls, resource):
+        """Factory: construct parameter from JSON resource.
+
+        :type resource: dict
+        :param resource: JSON mapping of parameter
+
+        :rtype: :class:`ArrayQueryParameter`
+        :returns: instance
+        """
+        array_type = resource['parameterType']['arrayType']['type']
+        if array_type == 'STRUCT':
+            return cls._from_api_repr_struct(resource)
+        return cls._from_api_repr_scalar(resource)
+
     def to_api_repr(self):
         """Construct JSON API representation for the parameter.
 
@@ -533,7 +583,7 @@ def to_api_repr(self):
         :returns: JSON mapping
         """
         values = self.values
-        if self.array_type == 'RECORD':
+        if self.array_type == 'RECORD' or self.array_type == 'STRUCT':
             reprs = [value.to_api_repr() for value in values]
             a_type = reprs[0]['parameterType']
             a_values = [repr_['parameterValue'] for repr_ in reprs]
@@ -556,6 +606,31 @@ def to_api_repr(self):
             resource['name'] = self.name
         return resource
 
+    def _key(self):
+        """A tuple key that uniquely describes this field.
+
+        Used to compute this instance's hashcode and evaluate equality.
+
+        Returns:
+            tuple: The contents of this :class:`ArrayQueryParameter`.
+        """
+        return (
+            self.name,
+            self.array_type.upper(),
+            self.values,
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, ArrayQueryParameter):
+            return NotImplemented
+        return self._key() == other._key()
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __repr__(self):
+        return 'ArrayQueryParameter{}'.format(self._key())
+
 
 class StructQueryParameter(AbstractQueryParameter):
     """Named / positional query parameters for struct values.
@@ -606,14 +681,32 @@ def from_api_repr(cls, resource):
         """
         name = resource.get('name')
         instance = cls(name)
+        type_resources = {}
         types = instance.struct_types
         for item in resource['parameterType']['structTypes']:
             types[item['name']] = item['type']['type']
+            type_resources[item['name']] = item['type']
         struct_values = resource['parameterValue']['structValues']
         for key, value in struct_values.items():
             type_ = types[key]
-            value = value['value']
-            converted = _CELLDATA_FROM_JSON[type_](value, None)
+            converted = None
+            if type_ == 'STRUCT':
+                struct_resource = {
+                    'name': key,
+                    'parameterType': type_resources[key],
+                    'parameterValue': value,
+                }
+                converted = StructQueryParameter.from_api_repr(struct_resource)
+            elif type_ == 'ARRAY':
+                struct_resource = {
+                    'name': key,
+                    'parameterType': type_resources[key],
+                    'parameterValue': value,
+                }
+                converted = ArrayQueryParameter.from_api_repr(struct_resource)
+            else:
+                value = value['value']
+                converted = _CELLDATA_FROM_JSON[type_](value, None)
             instance.struct_values[key] = converted
         return instance
 
@@ -651,6 +744,31 @@ def to_api_repr(self):
             resource['name'] = self.name
         return resource
 
+    def _key(self):
+        """A tuple key that uniquely describes this field.
+
+        Used to compute this instance's hashcode and evaluate equality.
+
+        Returns:
+            tuple: The contents of this :class:`ArrayQueryParameter`.
+        """
+        return (
+            self.name,
+            self.struct_types,
+            self.struct_values,
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, StructQueryParameter):
+            return NotImplemented
+        return self._key() == other._key()
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __repr__(self):
+        return 'StructQueryParameter{}'.format(self._key())
+
 
 class QueryParametersProperty(object):
     """Custom property type, holding query parameter instances."""


Test patch:
diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py
--- a/bigquery/tests/system.py
+++ b/bigquery/tests/system.py
@@ -887,6 +887,16 @@ def test_sync_query_w_query_params(self):
             name='friends', array_type='STRING',
             values=[phred_name, bharney_name])
         with_friends_param = StructQueryParameter(None, friends_param)
+        top_left_param = StructQueryParameter(
+            'top_left',
+            ScalarQueryParameter('x', 'INT64', 12),
+            ScalarQueryParameter('y', 'INT64', 102))
+        bottom_right_param = StructQueryParameter(
+            'bottom_right',
+            ScalarQueryParameter('x', 'INT64', 22),
+            ScalarQueryParameter('y', 'INT64', 92))
+        rectangle_param = StructQueryParameter(
+            'rectangle', top_left_param, bottom_right_param)
         examples = [
             {
                 'sql': 'SELECT @question',
@@ -943,6 +953,14 @@ def test_sync_query_w_query_params(self):
                 'expected': ({'_field_1': question, '_field_2': answer}),
                 'query_parameters': [struct_param],
             },
+            {
+                'sql':
+                    'SELECT '
+                    '((@rectangle.bottom_right.x - @rectangle.top_left.x) '
+                    '* (@rectangle.top_left.y - @rectangle.bottom_right.y))',
+                'expected': 100,
+                'query_parameters': [rectangle_param],
+            },
             {
                 'sql': 'SELECT ?',
                 'expected': [
diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py
--- a/bigquery/tests/unit/test__helpers.py
+++ b/bigquery/tests/unit/test__helpers.py
@@ -1098,6 +1098,55 @@ def test_to_api_repr_w_unknown_type(self):
         param = klass.positional(type_='UNKNOWN', value='unknown')
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test___eq___wrong_type(self):
+        field = self._make_one('test', 'STRING', 'value')
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___eq___name_mismatch(self):
+        field = self._make_one('test', 'STRING', 'value')
+        other = self._make_one('other', 'STRING', 'value')
+        self.assertNotEqual(field, other)
+
+    def test___eq___field_type_mismatch(self):
+        field = self._make_one('test', 'STRING', None)
+        other = self._make_one('test', 'INT64', None)
+        self.assertNotEqual(field, other)
+
+    def test___eq___value_mismatch(self):
+        field = self._make_one('test', 'STRING', 'hello')
+        other = self._make_one('test', 'STRING', 'world')
+        self.assertNotEqual(field, other)
+
+    def test___eq___hit(self):
+        field = self._make_one('test', 'STRING', 'gotcha')
+        other = self._make_one('test', 'STRING', 'gotcha')
+        self.assertEqual(field, other)
+
+    def test___ne___wrong_type(self):
+        field = self._make_one('toast', 'INT64', 13)
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___ne___same_value(self):
+        field1 = self._make_one('test', 'INT64', 12)
+        field2 = self._make_one('test', 'INT64', 12)
+        # unittest ``assertEqual`` uses ``==`` not ``!=``.
+        comparison_val = (field1 != field2)
+        self.assertFalse(comparison_val)
+
+    def test___ne___different_values(self):
+        field1 = self._make_one('test', 'INT64', 11)
+        field2 = self._make_one('test', 'INT64', 12)
+        self.assertNotEqual(field1, field2)
+
+    def test___repr__(self):
+        field1 = self._make_one('field1', 'STRING', 'value')
+        expected = "ScalarQueryParameter('field1', 'STRING', 'value')"
+        self.assertEqual(repr(field1), expected)
+
 
 def _make_subparam(name, type_, value):
     from google.cloud.bigquery._helpers import ScalarQueryParameter
@@ -1180,6 +1229,58 @@ def test_from_api_repr_wo_name(self):
         self.assertEqual(param.array_type, 'INT64')
         self.assertEqual(param.values, [1, 2])
 
+    def test_from_api_repr_w_struct_type(self):
+        from google.cloud.bigquery._helpers import StructQueryParameter
+
+        RESOURCE = {
+            'parameterType': {
+                'type': 'ARRAY',
+                'arrayType': {
+                    'type': 'STRUCT',
+                    'structTypes': [
+                        {
+                            'name': 'name',
+                            'type': {'type': 'STRING'},
+                        },
+                        {
+                            'name': 'age',
+                            'type': {'type': 'INT64'},
+                        },
+                    ],
+                },
+            },
+            'parameterValue': {
+                'arrayValues': [
+                    {
+                        'structValues': {
+                            'name': {'value': 'Phred Phlyntstone'},
+                            'age': {'value': '32'},
+                        },
+                    },
+                    {
+                        'structValues': {
+                            'name': {
+                                'value': 'Bharney Rhubbyl',
+                            },
+                            'age': {'value': '31'},
+                        },
+                    },
+                ],
+            },
+        }
+
+        klass = self._get_target_class()
+        param = klass.from_api_repr(RESOURCE)
+
+        phred = StructQueryParameter.positional(
+            _make_subparam('name', 'STRING', 'Phred Phlyntstone'),
+            _make_subparam('age', 'INT64', 32))
+        bharney = StructQueryParameter.positional(
+            _make_subparam('name', 'STRING', 'Bharney Rhubbyl'),
+            _make_subparam('age', 'INT64', 31))
+        self.assertEqual(param.array_type, 'STRUCT')
+        self.assertEqual(param.values, [phred, bharney])
+
     def test_to_api_repr_w_name(self):
         EXPECTED = {
             'name': 'foo',
@@ -1276,6 +1377,55 @@ def test_to_api_repr_w_record_type(self):
         param = klass.positional(array_type='RECORD', values=[struct])
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test___eq___wrong_type(self):
+        field = self._make_one('test', 'STRING', ['value'])
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___eq___name_mismatch(self):
+        field = self._make_one('field', 'STRING', ['value'])
+        other = self._make_one('other', 'STRING', ['value'])
+        self.assertNotEqual(field, other)
+
+    def test___eq___field_type_mismatch(self):
+        field = self._make_one('test', 'STRING', [])
+        other = self._make_one('test', 'INT64', [])
+        self.assertNotEqual(field, other)
+
+    def test___eq___value_mismatch(self):
+        field = self._make_one('test', 'STRING', ['hello'])
+        other = self._make_one('test', 'STRING', ['hello', 'world'])
+        self.assertNotEqual(field, other)
+
+    def test___eq___hit(self):
+        field = self._make_one('test', 'STRING', ['gotcha'])
+        other = self._make_one('test', 'STRING', ['gotcha'])
+        self.assertEqual(field, other)
+
+    def test___ne___wrong_type(self):
+        field = self._make_one('toast', 'INT64', [13])
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___ne___same_value(self):
+        field1 = self._make_one('test', 'INT64', [12])
+        field2 = self._make_one('test', 'INT64', [12])
+        # unittest ``assertEqual`` uses ``==`` not ``!=``.
+        comparison_val = (field1 != field2)
+        self.assertFalse(comparison_val)
+
+    def test___ne___different_values(self):
+        field1 = self._make_one('test', 'INT64', [11])
+        field2 = self._make_one('test', 'INT64', [12])
+        self.assertNotEqual(field1, field2)
+
+    def test___repr__(self):
+        field1 = self._make_one('field1', 'STRING', ['value'])
+        expected = "ArrayQueryParameter('field1', 'STRING', ['value'])"
+        self.assertEqual(repr(field1), expected)
+
 
 class Test_StructQueryParameter(unittest.TestCase):
 
@@ -1350,6 +1500,81 @@ def test_from_api_repr_wo_name(self):
         self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'})
         self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'})
 
+    def test_from_api_repr_w_nested_array(self):
+        from google.cloud.bigquery._helpers import ArrayQueryParameter
+
+        RESOURCE = {
+            'name': 'foo',
+            'parameterType': {
+                'type': 'STRUCT',
+                'structTypes': [
+                    {'name': 'bar', 'type': {'type': 'STRING'}},
+                    {'name': 'baz', 'type': {
+                        'type': 'ARRAY',
+                        'arrayType': {'type': 'INT64'},
+                    }},
+                ],
+            },
+            'parameterValue': {
+                'structValues': {
+                    'bar': {'value': 'abc'},
+                    'baz': {'arrayValues': [
+                        {'value': '123'},
+                        {'value': '456'},
+                    ]},
+                },
+            },
+        }
+        klass = self._get_target_class()
+        param = klass.from_api_repr(RESOURCE)
+        self.assertEqual(
+            param,
+            self._make_one(
+                'foo',
+                _make_subparam('bar', 'STRING', 'abc'),
+                ArrayQueryParameter('baz', 'INT64', [123, 456])))
+
+    def test_from_api_repr_w_nested_struct(self):
+        RESOURCE = {
+            'name': 'foo',
+            'parameterType': {
+                'type': 'STRUCT',
+                'structTypes': [
+                    {'name': 'bar', 'type': {'type': 'STRING'}},
+                    {'name': 'baz', 'type': {
+                        'type': 'STRUCT',
+                        'structTypes': [
+                            {'name': 'qux', 'type': {'type': 'INT64'}},
+                            {'name': 'spam', 'type': {'type': 'BOOL'}},
+                        ],
+                    }},
+                ],
+            },
+            'parameterValue': {
+                'structValues': {
+                    'bar': {'value': 'abc'},
+                    'baz': {'structValues': {
+                        'qux': {'value': '123'},
+                        'spam': {'value': 'true'},
+                    }},
+                },
+            },
+        }
+
+        klass = self._get_target_class()
+        param = klass.from_api_repr(RESOURCE)
+
+        expected = self._make_one(
+            'foo',
+            _make_subparam('bar', 'STRING', 'abc'),
+            self._make_one(
+                'baz',
+                _make_subparam('qux', 'INT64', 123),
+                _make_subparam('spam', 'BOOL', True)))
+        self.assertEqual(param.name, 'foo')
+        self.assertEqual(param.struct_types, expected.struct_types)
+        self.assertEqual(param.struct_values, expected.struct_values)
+
     def test_to_api_repr_w_name(self):
         EXPECTED = {
             'name': 'foo',
@@ -1457,6 +1682,72 @@ def test_to_api_repr_w_nested_struct(self):
         param = self._make_one('foo', scalar_1, sub)
         self.assertEqual(param.to_api_repr(), EXPECTED)
 
+    def test___eq___wrong_type(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'abc'))
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___eq___name_mismatch(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'abc'))
+        other = self._make_one(
+            'other ', _make_subparam('bar', 'STRING', 'abc'))
+        self.assertNotEqual(field, other)
+
+    def test___eq___field_type_mismatch(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', None))
+        other = self._make_one(
+            'test', _make_subparam('bar', 'INT64', None))
+        self.assertNotEqual(field, other)
+
+    def test___eq___value_mismatch(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'hello'))
+        other = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'world'))
+        self.assertNotEqual(field, other)
+
+    def test___eq___hit(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'gotcha'))
+        other = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'gotcha'))
+        self.assertEqual(field, other)
+
+    def test___ne___wrong_type(self):
+        field = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'hello'))
+        other = object()
+        self.assertNotEqual(field, other)
+        self.assertEqual(field, mock.ANY)
+
+    def test___ne___same_value(self):
+        field1 = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'hello'))
+        field2 = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'hello'))
+        # unittest ``assertEqual`` uses ``==`` not ``!=``.
+        comparison_val = (field1 != field2)
+        self.assertFalse(comparison_val)
+
+    def test___ne___different_values(self):
+        field1 = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'hello'))
+        field2 = self._make_one(
+            'test', _make_subparam('bar', 'STRING', 'world'))
+        self.assertNotEqual(field1, field2)
+
+    def test___repr__(self):
+        field1 = self._make_one(
+            'test', _make_subparam('field1', 'STRING', 'hello'))
+        got = repr(field1)
+        self.assertIn('StructQueryParameter', got)
+        self.assertIn("'field1', 'STRING'", got)
+        self.assertIn("'field1': 'hello'", got)
+
 
 class Test_QueryParametersProperty(unittest.TestCase):