Repository: GoogleCloudPlatform/professional-services

Issue:
asset-inventory Failed : google.api_core.exceptions.BadRequest: 400 Too many fields: 10018 [while running 'load_to_bigquery/load_to_bigquery']
Hi there,
DataFlow Job is failing since January 29th. I guess something has changed on google side?
Can you correct the pipeline please?
I can give you a read access to our project if needed.
here is the error:
```
"An exception was raised when trying to execute the workitem 5435749220349416924 : Traceback (most recent call last):
  File "apache_beam/runners/common.py", line 1246, in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
  File "apache_beam/runners/common.py", line 514, in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
  File "apache_beam/runners/common.py", line 520, in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
  File "./asset_inventory/import_pipeline.py", line 422, in finish_bundle
    raise e
  File "./asset_inventory/import_pipeline.py", line 419, in finish_bundle
    load_job.result()
  File "/usr/local/lib/python3.7/site-packages/google/cloud/bigquery/job.py", line 733, in result
    return super(_AsyncJob, self).result(timeout=timeout)
  File "/usr/local/lib/python3.7/site-packages/google/api_core/future/polling.py", line 130, in result
    raise self._exception
google.api_core.exceptions.BadRequest: 400 Too many fields: 10018

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/dataflow_worker/batchworker.py", line 649, in do_work
    work_executor.execute()
  File "/usr/local/lib/python3.7/site-packages/dataflow_worker/executor.py", line 181, in execute
    op.finish()
  File "apache_beam/runners/worker/operations.py", line 730, in apache_beam.runners.worker.operations.DoOperation.finish
  File "apache_beam/runners/worker/operations.py", line 732, in apache_beam.runners.worker.operations.DoOperation.finish
  File "apache_beam/runners/worker/operations.py", line 733, in apache_beam.runners.worker.operations.DoOperation.finish
  File "apache_beam/runners/common.py", line 1267, in apache_beam.runners.common.DoFnRunner.finish
  File "apache_beam/runners/common.py", line 1248, in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
  File "apache_beam/runners/common.py", line 1294, in apache_beam.runners.common.DoFnRunner._reraise_augmented
  File "/usr/local/lib/python3.7/site-packages/future/utils/__init__.py", line 446, in raise_with_traceback
    raise exc.with_traceback(traceback)
  File "apache_beam/runners/common.py", line 1246, in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
  File "apache_beam/runners/common.py", line 514, in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
  File "apache_beam/runners/common.py", line 520, in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
  File "./asset_inventory/import_pipeline.py", line 422, in finish_bundle
    raise e
  File "./asset_inventory/import_pipeline.py", line 419, in finish_bundle
    load_job.result()
  File "/usr/local/lib/python3.7/site-packages/google/cloud/bigquery/job.py", line 733, in result
    return super(_AsyncJob, self).result(timeout=timeout)
  File "/usr/local/lib/python3.7/site-packages/google/api_core/future/polling.py", line 130, in result
    raise self._exception
google.api_core.exceptions.BadRequest: 400 Too many fields: 10018 [while running 'load_to_bigquery/load_to_bigquery']
"


```


Discussion:
@bmenasha PTAL

Resolving patch:
diff --git a/tools/asset-inventory/asset_inventory/bigquery_schema.py b/tools/asset-inventory/asset_inventory/bigquery_schema.py
--- a/tools/asset-inventory/asset_inventory/bigquery_schema.py
+++ b/tools/asset-inventory/asset_inventory/bigquery_schema.py
@@ -124,6 +124,13 @@ def get_field_by_name(fields, field_name):
     return None, None
 
 
+def is_additonal_properties(fields):
+    return fields and len(fields) == 2 and all(
+        (f.get('name', None) == 'name'
+         and f.get('description', None) == 'additionalProperties name')
+        or (f.get('name', None) == 'value') for f in fields)
+
+
 def _merge_fields(destination_field, source_field):
     """Combines two SchemaField like dicts.
 
@@ -143,19 +150,46 @@ def _merge_fields(destination_field, source_field):
     sd = source_field.get('description', None)
     dft = destination_field.get('field_type', None)
     sft = source_field.get('field_type', None)
-    # use the  field with more information.
+    # use the field with more information.
     if ((not dd and sd) or (sd and dd and len(dd) < len(sd))):
         field['description'] = sd
         field['field_type'] = sft
-    # use the less specific type.
+    # use the less specific type. and join fields
     elif ((dft != 'RECORD' and dft != 'STRING') and sft == 'STRING'):
         field['field_type'] = sft
-    df = destination_field.get('fields', [])
+
+    # https://github.com/GoogleCloudPlatform/professional-services/issues/614
+    # Use the schema with the additonalProperties overrides. See
+    # api_schema._get_properties_map_field_list which creates the
+    # additionalProperties RECORD type and enforce_schema_data_types for where
+    # documents of type RECORD are converted to the REPEATED additonalProperties
+    # name value pairs.
+    def merge_additional_properties_fields(apf, fields):
+        i, value_field = get_field_by_name(apf, 'value')
+        for f in fields:
+            if f.get('name', None) not in ('name', 'value'):
+                value_field = _merge_fields(value_field, f)
+        apf[i] = value_field
+
     sf = source_field.get('fields', [])
-    merged_fields = _merge_schema(df, sf)
-    # recursivly merge nested fields.
-    if merged_fields != df:
-        field['fields'] = merged_fields
+    df = destination_field.get('fields', [])
+    if is_additonal_properties(sf) and not is_additonal_properties(df):
+        field['mode'] = 'REPEATED'
+        sf = copy.deepcopy(sf)
+        merge_additional_properties_fields(sf, df)
+        field['fields'] = sf
+    elif is_additonal_properties(df) and not is_additonal_properties(sf):
+        field['mode'] = 'REPEATED'
+        merge_additional_properties_fields(df, sf)
+        field['fields'] = df
+    elif is_additonal_properties(df) and is_additonal_properties(sf):
+        field['mode'] = 'REPEATED'
+        merge_additional_properties_fields(df, sf)
+        field['fields'] = df
+    else:
+        mf = _merge_schema(df, sf)
+        if mf:
+            field['fields'] = mf
     return field
 
 
@@ -448,6 +482,7 @@ def enforce_schema_data_types(resource, schema):
             if field.get('mode', 'NULLABLE') == 'REPEATED':
                 # satisfy array condition by converting dict into
                 # repeated name value records.
+                # this handles any 'additonalProperties' types.
                 if (field['field_type'] == 'RECORD' and
                     isinstance(resource_value, dict)):
                     resource_value = [{'name': key, 'value': val}


Test patch:
diff --git a/tools/asset-inventory/tests/test_bigquery_schema.py b/tools/asset-inventory/tests/test_bigquery_schema.py
--- a/tools/asset-inventory/tests/test_bigquery_schema.py
+++ b/tools/asset-inventory/tests/test_bigquery_schema.py
@@ -22,6 +22,7 @@
 
 class TestBigQuerySchema(unittest.TestCase):
 
+
     def test_record(self):
         document = {'record_field': {'string_field': 'string_value'}}
         schema = bigquery_schema.translate_json_to_schema(
@@ -331,6 +332,106 @@ def test_prune_max_properties(self):
         self.assertEqual(len(sanitized), 10000)
         self.assertNotIn('z', sanitized)
 
+    def test_addtional_properties_merge_schema_simple(self):
+        rest_schema = [
+            {'name': 'property_1',
+             'field_type': 'STRING',
+             'description': 'description-1',
+             'mode': 'NULLABLE'
+            },
+            {'name': 'property_2',
+             'field_type': 'RECORD',
+             'description': 'description-2',
+             'mode': 'REPEATED',
+             'fields': [{'name': 'name',
+                         'field_type': 'STRING',
+                         'description': 'additionalProperties name',
+                         'mode': 'NULLABLE'},
+                        {'name': 'value',
+                         'field_type': 'STRING',
+                         'description': 'description-2.',
+                         'mode': 'NULLABLE'}]}]
+
+        document = {
+            'property_1': 'value_1',
+            'property_2': {
+                'add_prop_1': 'add_value_1',
+                'add_prop_2': 'add_value_2'
+                },
+            'property_3': 'value_3'
+        }
+
+        document_schema = bigquery_schema.translate_json_to_schema(
+            document)
+
+        self.assertEqual(
+            bigquery_schema.merge_schemas(
+                [rest_schema, document_schema]
+            ),
+            rest_schema + [{'name': 'property_3',
+                            'field_type': 'STRING',
+                            'mode': 'NULLABLE'
+                           }])
+
+    def test_addtional_properties_merge_schema_object(self):
+        self.maxDiff = None
+        rest_schema = [
+            {'name': 'property_1',
+             'field_type': 'STRING',
+             'description': 'description-1',
+             'mode': 'NULLABLE'
+            },
+            {'name': 'property_2',
+             'field_type': 'RECORD',
+             'description': 'description-2',
+             'mode': 'REPEATED',
+             'fields': [{'name': 'name',
+                         'field_type': 'STRING',
+                         'description': 'additionalProperties name',
+                         'mode': 'NULLABLE'},
+                        {'name': 'value',
+                         'field_type': 'RECORD',
+                         'mode': 'NULLABLE'}]}]
+
+        document = {
+            'property_1': 'value_1',
+            'property_2': {
+                'add_prop_1': {'key_1': 1},
+                'add_prop_2': {'key_1': 2}
+                },
+            'property_3': 'value_3'
+        }
+
+        document_schema = bigquery_schema.translate_json_to_schema(
+            document)
+
+        self.assertEqual(
+            bigquery_schema.merge_schemas(
+                [rest_schema, document_schema]
+            ),
+            [{'name': 'property_1',
+              'field_type': 'STRING',
+              'description': 'description-1',
+              'mode': 'NULLABLE'
+             },
+             {'name': 'property_2',
+              'field_type': 'RECORD',
+              'description': 'description-2',
+              'mode': 'REPEATED',
+              'fields': [{'name': 'name',
+                          'field_type': 'STRING',
+                          'description': 'additionalProperties name',
+                          'mode': 'NULLABLE'},
+                         {'name': 'value',
+                          'field_type': 'RECORD',
+                          'mode': 'NULLABLE',
+                          'fields': [{'name': 'key_1',
+                                      'field_type': 'NUMERIC',
+                                      'mode': 'NULLABLE'}]}]
+             },
+             {'name': 'property_3',
+              'field_type': 'STRING',
+              'mode': 'NULLABLE'}])
 
 
 if __name__ == '__main__':