Repository: googleapis/google-cloud-python

Issue:
BigTable: row_data _validateChunk performance
Looking at the performance graph for reading a large number (1000) of rows, each row having 10 cells in one column family, the validation of chunks takes a large percentage of time (>25%), as measured by cProfile.  Areas for tuning are identified below.

**Validation of timestamp_micros ordering** 
While a row is being read, a validation is made to make sure that cells for one qualifier are in timestamp_micros order (most recent first):
```
    def _validate_chunk_row_in_progress(self, chunk):
        ...
        previous = self._previous_cell
        _raise_if(self._same_as_previous(chunk) and
                  chunk.timestamp_micros <= previous.timestamp_micros)

    def _same_as_previous(self, chunk):
        previous = self._previous_cell
        return (chunk.row_key == previous.row_key and
                chunk.family_name == previous.family_name and
                chunk.qualifier == previous.qualifier and
                chunk.labels == previous.labels)
```
This validation may be extraneous because:

1. A RowFilter may have been applied - CellsColumnLimitFilter(1) - which will limit the result to the cell with the most recent timestamp.
2. There may only be one cell value for the qualifier.
3. The BigTable server may ALWAYS be providing the cells in timestamp order.

Two possible tunings:

1. Remove the validation if the condition is unlikely to occur in the ReadRowResponse.
2. If the validation is kept, it could be performed only when qualifier cell lists with length > 1 are detected AFTER the chunks for the qualifier are consumed.


Discussion:

Resolving patch:
diff --git a/bigtable/google/cloud/bigtable/row_data.py b/bigtable/google/cloud/bigtable/row_data.py
--- a/bigtable/google/cloud/bigtable/row_data.py
+++ b/bigtable/google/cloud/bigtable/row_data.py
@@ -48,10 +48,10 @@ class Cell(object):
     :param labels: (Optional) List of strings. Labels applied to the cell.
     """
 
-    def __init__(self, value, timestamp_micros, labels=()):
+    def __init__(self, value, timestamp_micros, labels=None):
         self.value = value
         self.timestamp_micros = timestamp_micros
-        self.labels = list(labels)
+        self.labels = list(labels) if labels is not None else []
 
     @classmethod
     def from_pb(cls, cell_pb):
@@ -374,6 +374,15 @@ class YieldRowsData(object):
     ROW_IN_PROGRESS = 'Row in progress'  # Some cells complete for row
     CELL_IN_PROGRESS = 'Cell in progress'  # Incomplete cell for row
 
+    STATE_START = 0
+    STATE_NEW_ROW = 1
+    STATE_ROW_IN_PROGRESS = 2
+    STATE_CELL_IN_PROGRESS = 3
+
+    read_states = {STATE_START: START, STATE_NEW_ROW: NEW_ROW,
+                   STATE_ROW_IN_PROGRESS: ROW_IN_PROGRESS,
+                   STATE_CELL_IN_PROGRESS: CELL_IN_PROGRESS}
+
     def __init__(self, read_method, request):
         # Counter for responses pulled from iterator
         self._counter = 0
@@ -400,17 +409,24 @@ def state(self):
         :returns:  name of state corresponding to current row / chunk
                    processing.
         """
+        return self.read_states[self._state]
+
+    @property
+    def _state(self):
+        """State machine state.
+        :rtype: int
+        :returns:  id of state corresponding to currrent row / chunk
+                   processing.
+        """
+        if self._previous_cell is not None:
+            return self.STATE_ROW_IN_PROGRESS
         if self.last_scanned_row_key is None:
-            return self.START
+            return self.STATE_START
         if self._row is None:
-            assert self._cell is None
-            assert self._previous_cell is None
-            return self.NEW_ROW
+            return self.STATE_NEW_ROW
         if self._cell is not None:
-            return self.CELL_IN_PROGRESS
-        if self._previous_cell is not None:
-            return self.ROW_IN_PROGRESS
-        return self.NEW_ROW  # row added, no chunk yet processed
+            return self.STATE_CELL_IN_PROGRESS
+        return self.STATE_NEW_ROW  # row added, no chunk yet processed
 
     def cancel(self):
         """Cancels the iterator, closing the stream."""
@@ -470,33 +486,37 @@ def read_rows(self):
 
             for chunk in response.chunks:
 
-                self._validate_chunk(chunk)
-
                 if chunk.reset_row:
+                    self._validate_chunk_reset_row(chunk)
                     row = self._row = None
                     cell = self._cell = self._previous_cell = None
                     continue
 
-                if row is None:
-                    row = self._row = PartialRowData(chunk.row_key)
-
                 if cell is None:
-                    qualifier = None
-                    if chunk.HasField('qualifier'):
-                        qualifier = chunk.qualifier.value
+                    qualifier = chunk.qualifier.value
+                    if qualifier == b'' and not chunk.HasField('qualifier'):
+                        qualifier = None
 
-                    cell = self._cell = PartialCellData(
+                    cell = PartialCellData(
                         chunk.row_key,
                         chunk.family_name.value,
                         qualifier,
                         chunk.timestamp_micros,
                         chunk.labels,
                         chunk.value)
+                    self._validate_cell_data(cell)
+                    self._cell = cell
                     self._copy_from_previous(cell)
                 else:
                     cell.append_value(chunk.value)
 
+                if row is None:
+                    row = self._row = PartialRowData(cell.row_key)
+
                 if chunk.commit_row:
+                    if chunk.value_size > 0:
+                        raise InvalidChunk()
+
                     self._save_current_cell()
 
                     yield self._row
@@ -511,94 +531,66 @@ def read_rows(self):
                     self._save_current_cell()
                     cell = None
 
-    @staticmethod
-    def _validate_chunk_status(chunk):
-        """Helper for :meth:`_validate_chunk_row_in_progress`, etc."""
-        # No reseet with other keys
-        if chunk.reset_row:
-            _raise_if(chunk.row_key)
-            _raise_if(chunk.HasField('family_name'))
-            _raise_if(chunk.HasField('qualifier'))
-            _raise_if(chunk.timestamp_micros)
-            _raise_if(chunk.labels)
-            _raise_if(chunk.value_size)
-            _raise_if(chunk.value)
-        # No commit with value size
-        _raise_if(chunk.commit_row and chunk.value_size > 0)
-        # No negative value_size (inferred as a general constraint).
-        _raise_if(chunk.value_size < 0)
-
-    def _validate_chunk_new_row(self, chunk):
-        """Helper for :meth:`_validate_chunk`."""
-        assert self.state == self.NEW_ROW
-        _raise_if(chunk.reset_row)
-        _raise_if(not chunk.row_key)
-        _raise_if(not chunk.family_name)
-        _raise_if(not chunk.qualifier)
-        # This constraint is not enforced in the Go example.
-        _raise_if(chunk.value_size > 0 and chunk.commit_row is not False)
-        # This constraint is from the Go example, not the spec.
-        _raise_if(self._previous_row is not None and
-                  chunk.row_key <= self._previous_row.row_key)
-
-    def _same_as_previous(self, chunk):
-        """Helper for :meth:`_validate_chunk_row_in_progress`"""
-        previous = self._previous_cell
-        return (chunk.row_key == previous.row_key and
-                chunk.family_name == previous.family_name and
-                chunk.qualifier == previous.qualifier and
-                chunk.labels == previous.labels)
-
-    def _validate_chunk_row_in_progress(self, chunk):
-        """Helper for :meth:`_validate_chunk`"""
-        assert self.state == self.ROW_IN_PROGRESS
-        self._validate_chunk_status(chunk)
-        _raise_if(chunk.row_key and
-                  chunk.row_key != self._row.row_key)
-        _raise_if(chunk.HasField('family_name') and
-                  not chunk.HasField('qualifier'))
-        previous = self._previous_cell
-        _raise_if(self._same_as_previous(chunk) and
-                  chunk.timestamp_micros <= previous.timestamp_micros)
-
-    def _validate_chunk_cell_in_progress(self, chunk):
-        """Helper for :meth:`_validate_chunk`"""
-        assert self.state == self.CELL_IN_PROGRESS
-        self._validate_chunk_status(chunk)
-        self._copy_from_current(chunk)
-
-    def _validate_chunk(self, chunk):
-        """Helper for :meth:`consume_next`."""
-        if self.state == self.NEW_ROW:
-            self._validate_chunk_new_row(chunk)
-        if self.state == self.ROW_IN_PROGRESS:
-            self._validate_chunk_row_in_progress(chunk)
-        if self.state == self.CELL_IN_PROGRESS:
-            self._validate_chunk_cell_in_progress(chunk)
+    def _validate_cell_data(self, cell):
+        if self._state == self.STATE_ROW_IN_PROGRESS:
+            self._validate_cell_data_row_in_progress(cell)
+        if self._state == self.STATE_NEW_ROW:
+            self._validate_cell_data_new_row(cell)
+        if self._state == self.STATE_CELL_IN_PROGRESS:
+            self._copy_from_current(cell)
+
+    def _validate_cell_data_new_row(self, cell):
+        if (not cell.row_key or
+                not cell.family_name or
+                cell.qualifier is None):
+            raise InvalidChunk()
+
+        if (self._previous_row is not None and
+                cell.row_key <= self._previous_row.row_key):
+            raise InvalidChunk()
+
+    def _validate_cell_data_row_in_progress(self, cell):
+        if ((cell.row_key and
+             cell.row_key != self._row.row_key) or
+                (cell.family_name and cell.qualifier is None)):
+            raise InvalidChunk()
+
+    def _validate_chunk_reset_row(self, chunk):
+        # No reset for new row
+        _raise_if(self._state == self.STATE_NEW_ROW)
+
+        # No reset with other keys
+        _raise_if(chunk.row_key)
+        _raise_if(chunk.HasField('family_name'))
+        _raise_if(chunk.HasField('qualifier'))
+        _raise_if(chunk.timestamp_micros)
+        _raise_if(chunk.labels)
+        _raise_if(chunk.value_size)
+        _raise_if(chunk.value)
 
     def _save_current_cell(self):
         """Helper for :meth:`consume_next`."""
         row, cell = self._row, self._cell
         family = row._cells.setdefault(cell.family_name, {})
         qualified = family.setdefault(cell.qualifier, [])
-        complete = Cell.from_pb(self._cell)
+        complete = Cell.from_pb(cell)
         qualified.append(complete)
         self._cell, self._previous_cell = None, cell
 
-    def _copy_from_current(self, chunk):
-        """Helper for :meth:`consume_next`."""
+    def _copy_from_current(self, cell):
         current = self._cell
         if current is not None:
-            if not chunk.row_key:
-                chunk.row_key = current.row_key
-            if not chunk.HasField('family_name'):
-                chunk.family_name.value = current.family_name
-            if not chunk.HasField('qualifier'):
-                chunk.qualifier.value = current.qualifier
-            if not chunk.timestamp_micros:
-                chunk.timestamp_micros = current.timestamp_micros
-            if not chunk.labels:
-                chunk.labels.extend(current.labels)
+            if not cell.row_key:
+                cell.row_key = current.row_key
+            if not cell.family_name:
+                cell.family_name = current.family_name
+                # NOTE: ``cell.qualifier`` **can** be empty string.
+            if cell.qualifier is None:
+                cell.qualifier = current.qualifier
+            if not cell.timestamp_micros:
+                cell.timestamp_micros = current.timestamp_micros
+            if not cell.labels:
+                cell.labels.extend(current.labels)
 
     def _copy_from_previous(self, cell):
         """Helper for :meth:`consume_next`."""


Test patch:
diff --git a/bigtable/tests/unit/test_row_data.py b/bigtable/tests/unit/test_row_data.py
--- a/bigtable/tests/unit/test_row_data.py
+++ b/bigtable/tests/unit/test_row_data.py
@@ -17,6 +17,41 @@
 
 import mock
 
+from ._testing import _make_credentials
+
+
+class MultiCallableStub(object):
+    """Stub for the grpc.UnaryUnaryMultiCallable interface."""
+
+    def __init__(self, method, channel_stub):
+        self.method = method
+        self.channel_stub = channel_stub
+
+    def __call__(self, request, timeout=None, metadata=None, credentials=None):
+        self.channel_stub.requests.append((self.method, request))
+
+        return self.channel_stub.responses.pop()
+
+
+class ChannelStub(object):
+    """Stub for the grpc.Channel interface."""
+
+    def __init__(self, responses=[]):
+        self.responses = responses
+        self.requests = []
+
+    def unary_unary(self,
+                    method,
+                    request_serializer=None,
+                    response_deserializer=None):
+        return MultiCallableStub(method, self)
+
+    def unary_stream(self,
+                     method,
+                     request_serializer=None,
+                     response_deserializer=None):
+        return MultiCallableStub(method, self)
+
 
 class TestCell(unittest.TestCase):
     timestamp_micros = 18738724000  # Make sure millis granularity
@@ -369,6 +404,15 @@ def _get_target_class():
     def _make_one(self, *args, **kwargs):
         return self._get_target_class()(*args, **kwargs)
 
+    @staticmethod
+    def _get_target_client_class():
+        from google.cloud.bigtable.client import Client
+
+        return Client
+
+    def _make_client(self, *args, **kwargs):
+        return self._get_target_client_class()(*args, **kwargs)
+
     def test_state_start(self):
         client = _Client()
         iterator = _MockCancellableIterator()
@@ -379,13 +423,36 @@ def test_state_start(self):
         self.assertEqual(yrd.state, yrd.START)
 
     def test_state_new_row_w_row(self):
-        client = _Client()
-        iterator = _MockCancellableIterator()
-        client._data_stub = mock.MagicMock()
-        client._data_stub.ReadRows.side_effect = [iterator]
+        from google.cloud.bigtable_v2.gapic import bigtable_client
+
+        chunk = _ReadRowsResponseCellChunkPB(
+            row_key=self.ROW_KEY,
+            family_name=self.FAMILY_NAME,
+            qualifier=self.QUALIFIER,
+            timestamp_micros=self.TIMESTAMP_MICROS,
+            value=self.VALUE,
+            commit_row=True,
+        )
+        chunks = [chunk]
+
+        response = _ReadRowsResponseV2(chunks)
+        iterator = _MockCancellableIterator(response)
+        channel = ChannelStub(responses=[iterator])
+        data_api = bigtable_client.BigtableClient(channel=channel)
+        credentials = _make_credentials()
+        client = self._make_client(project='project-id',
+                                   credentials=credentials, admin=True)
+        client._table_data_client = data_api
         request = object()
-        yrd = self._make_one(client._data_stub.ReadRows, request)
-        yrd.last_scanned_row_key = ''
+        yrd = self._make_one(
+            client._table_data_client.bigtable_stub.ReadRows, request)
+        yrd._response_iterator = iterator
+        yrd._last_scanned_row_key = ''
+        rows = [row for row in yrd.read_rows()]
+
+        result = rows[0]
+        self.assertEqual(result.row_key, self.ROW_KEY)
+
         yrd._row = object()
         self.assertEqual(yrd.state, yrd.NEW_ROW)
 
@@ -441,6 +508,22 @@ def test__copy_from_current_blank(self):
         self.assertEqual(chunk.timestamp_micros, TIMESTAMP_MICROS)
         self.assertEqual(chunk.labels, LABELS)
 
+    def test__copy_from_current_empty_chunk(self):
+        client = _Client()
+        client._data_stub = mock.MagicMock()
+        request = object()
+        yrd = self._make_one(client._data_stub.ReadRows, request)
+        yrd._cell = _PartialCellData()
+        yrd._cell.qualifier = b''
+        chunks = _generate_cell_chunks([''])
+        chunk = chunks[0]
+        yrd._copy_from_current(chunk)
+        self.assertEqual(chunk.row_key, b'')
+        self.assertEqual(chunk.family_name.value, '')
+        self.assertEqual(chunk.qualifier.value, b'')
+        self.assertEqual(chunk.timestamp_micros, 0)
+        self.assertEqual(chunk.labels, [])
+
     def test__copy_from_previous_unset(self):
         client = _Client()
         client._data_stub = mock.MagicMock()
@@ -448,7 +531,7 @@ def test__copy_from_previous_unset(self):
         yrd = self._make_one(client._data_stub.ReadRows, request)
         cell = _PartialCellData()
         yrd._copy_from_previous(cell)
-        self.assertEqual(cell.row_key, '')
+        self.assertEqual(cell.row_key, b'')
         self.assertEqual(cell.family_name, u'')
         self.assertIsNone(cell.qualifier)
         self.assertEqual(cell.timestamp_micros, 0)
@@ -544,6 +627,50 @@ def test_invalid_empty_chunk(self):
         with self.assertRaises(InvalidChunk):
             self._consume_all(yrd)
 
+    def test_state_cell_in_progress(self):
+        LABELS = ['L1', 'L2']
+
+        client = _Client()
+        chunk = _ReadRowsResponseCellChunkPB(
+            row_key=self.ROW_KEY,
+            family_name=self.FAMILY_NAME,
+            qualifier=self.QUALIFIER,
+            timestamp_micros=self.TIMESTAMP_MICROS,
+            value=self.VALUE,
+            commit_row=True,
+        )
+        chunks = [chunk]
+        response = _ReadRowsResponseV2(chunks)
+        iterator = _MockCancellableIterator(response)
+        client._data_stub = mock.MagicMock()
+        client._data_stub.ReadRows.side_effect = [iterator]
+        request = object()
+        yrd = self._make_one(client._data_stub.ReadRows, request)
+        self._consume_all(yrd)
+        yrd._last_scanned_row_key = ''
+        yrd._row = object()
+        cell = _PartialCellData(
+            row_key=self.ROW_KEY,
+            family_name=self.FAMILY_NAME,
+            qualifier=self.QUALIFIER,
+            timestamp_micros=self.TIMESTAMP_MICROS,
+            labels=LABELS
+        )
+
+        yrd._cell = cell
+        more_cell_data = _PartialCellData(
+            value=self.VALUE
+        )
+
+        yrd._validate_cell_data(more_cell_data)
+
+        self.assertEqual(more_cell_data.row_key, self.ROW_KEY)
+        self.assertEqual(more_cell_data.family_name, self.FAMILY_NAME)
+        self.assertEqual(more_cell_data.qualifier, self.QUALIFIER)
+        self.assertEqual(more_cell_data.timestamp_micros,
+                         self.TIMESTAMP_MICROS)
+        self.assertEqual(more_cell_data.labels, LABELS)
+
     def test_yield_rows_data(self):
         client = _Client()
 
@@ -848,7 +975,7 @@ def next(self):
 
 class _PartialCellData(object):
 
-    row_key = ''
+    row_key = b''
     family_name = u''
     qualifier = None
     timestamp_micros = 0