Repository: googleapis/python-bigquery

Issue:
BigQuery: complex repeated schema fields are slow
When converting rows to JSON, repeated schema fields are deepcopied causing noticeable slowdowns in complex schemas.
Since conversion is performed recursively, complex schemas containing multiple levels of repeated fields (e.g. repeated list of structs each containing multiple fields of repeated structs) are copied multiple times when converting a single row - causing even further slowdown due to redundant copying. 

I've created a proposal for removing the deepcopy operation. Will reference relevant PR.


Discussion:
Just to demonstrate the issue, these are the sort of profiling results one may expect when inserting a row into a complex-schema table:
![image](https://user-images.githubusercontent.com/2963806/72828771-05465100-3c86-11ea-9098-d1fa380f1f77.png)


Resolving patch:
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
--- a/google/cloud/bigquery/_helpers.py
+++ b/google/cloud/bigquery/_helpers.py
@@ -15,7 +15,6 @@
 """Shared helper functions for BigQuery API classes."""
 
 import base64
-import copy
 import datetime
 import decimal
 import re
@@ -397,13 +396,9 @@ def _repeated_field_to_json(field, row_value):
     Returns:
         List[Any]: A list of JSON-serializable objects.
     """
-    # Remove the REPEATED, but keep the other fields. This allows us to process
-    # each item as if it were a top-level field.
-    item_field = copy.deepcopy(field)
-    item_field._mode = "NULLABLE"
     values = []
     for item in row_value:
-        values.append(_field_to_json(item_field, item))
+        values.append(_single_field_to_json(field, item))
     return values
 
 
@@ -462,6 +457,33 @@ def _record_field_to_json(fields, row_value):
     return record
 
 
+def _single_field_to_json(field, row_value):
+    """Convert a single field into JSON-serializable values.
+
+    Ignores mode so that this can function for ARRAY / REPEATING fields
+    without requiring a deepcopy of the field. See:
+    https://github.com/googleapis/python-bigquery/issues/6
+
+    Args:
+        field (google.cloud.bigquery.schema.SchemaField):
+            The SchemaField to use for type conversion and field name.
+
+        row_value (Any):
+            Scalar or Struct to be inserted. The type
+            is inferred from the SchemaField's field_type.
+
+    Returns:
+        Any: A JSON-serializable object.
+    """
+    if row_value is None:
+        return None
+
+    if field.field_type == "RECORD":
+        return _record_field_to_json(field.fields, row_value)
+
+    return _scalar_field_to_json(field, row_value)
+
+
 def _field_to_json(field, row_value):
     """Convert a field into JSON-serializable values.
 
@@ -483,10 +505,7 @@ def _field_to_json(field, row_value):
     if field.mode == "REPEATED":
         return _repeated_field_to_json(field, row_value)
 
-    if field.field_type == "RECORD":
-        return _record_field_to_json(field.fields, row_value)
-
-    return _scalar_field_to_json(field, row_value)
+    return _single_field_to_json(field, row_value)
 
 
 def _snake_to_camel_case(value):


Test patch:
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
--- a/tests/unit/test__helpers.py
+++ b/tests/unit/test__helpers.py
@@ -806,6 +806,41 @@ def test_w_known_field_type(self):
         self.assertEqual(converted, str(original))
 
 
+class Test_single_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _single_field_to_json
+
+        return _single_field_to_json(field, value)
+
+    def test_w_none(self):
+        field = _make_field("INT64")
+        original = None
+        converted = self._call_fut(field, original)
+        self.assertIsNone(converted)
+
+    def test_w_record(self):
+        subfields = [
+            _make_field("INT64", name="one"),
+            _make_field("STRING", name="two"),
+        ]
+        field = _make_field("RECORD", fields=subfields)
+        original = {"one": 42, "two": "two"}
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, {"one": "42", "two": "two"})
+
+    def test_w_scalar(self):
+        field = _make_field("INT64")
+        original = 42
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, str(original))
+
+    def test_w_scalar_ignores_mode(self):
+        field = _make_field("STRING", mode="REPEATED")
+        original = "hello world"
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, original)
+
+
 class Test_repeated_field_to_json(unittest.TestCase):
     def _call_fut(self, field, value):
         from google.cloud.bigquery._helpers import _repeated_field_to_json