Repository: googleapis/google-cloud-python

Issue:
Firestore: Field path keys are not expanded for dicts used as cursors.
Copied from @torresalmonte's issue at https://github.com/firebase/firebase-admin-python/issues/281

```
l050ZZ1bAsBddd7hE1GD => {'wordcount': {'page3': 130, 'page2': 120, 'page1': 100}, 'availability_date': DatetimeWithNanoseconds(2019, 3, 4, 6, 0, tzinfo=<UTC>), 'count': 100}
JCn6BxwfEDIvirnkMyOf => {'wordcount': {'page3': 180, 'page2': 120, 'page1': 150}, 'availability_date': DatetimeWithNanoseconds(2019, 3, 4, 6, 0, tzinfo=<UTC>), 'count': 150}
nfW0ybNz4WPGWtikfVK2 => {'count': 200, 'wordcount': {'page3': 50, 'page2': 100, 'page1': 200}}
last value: 200
Traceback (most recent call last):
  File "/home/ctorres/.local/lib/python3.5/site-packages/google/cloud/firestore_v1/query.py", line 642, in _normalize_cursor
    values.append(field_path_module.get_nested_value(order_key, data))
  File "/home/ctorres/.local/lib/python3.5/site-packages/google/cloud/firestore_v1/field_path.py", line 243, in get_nested_value
    raise KeyError(msg)
KeyError: "'wordcount' is not contained in the data"

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "firestore_pagination.py", line 28, in <module>
    docs_list = list(docs)
  File "/home/ctorres/.local/lib/python3.5/site-packages/google/cloud/firestore_v1/query.py", line 736, in stream
    self._to_protobuf(),
  File "/home/ctorres/.local/lib/python3.5/site-packages/google/cloud/firestore_v1/query.py", line 675, in _to_protobuf
    start_at = self._normalize_cursor(self._start_at, orders)
  File "/home/ctorres/.local/lib/python3.5/site-packages/google/cloud/firestore_v1/query.py", line 645, in _normalize_cursor
    raise ValueError(msg)
ValueError: The "order by" field path 'wordcount.page1' is not present in the cursor data {'wordcount.page1': 200}. All fields sent to ``order_by()`` must be present in the fields if passed to one of ``start_at()`` / ``start_after()`` / ``end_before()`` / ``end_at()`` to define a cursor.
```

For repo steps, please look at the original GitHub issue.


Discussion:

Resolving patch:
diff --git a/firestore/google/cloud/firestore_v1/query.py b/firestore/google/cloud/firestore_v1/query.py
--- a/firestore/google/cloud/firestore_v1/query.py
+++ b/firestore/google/cloud/firestore_v1/query.py
@@ -666,7 +666,12 @@ def _normalize_cursor(self, cursor, orders):
             data = document_fields
             for order_key in order_keys:
                 try:
-                    values.append(field_path_module.get_nested_value(order_key, data))
+                    if order_key in data:
+                        values.append(data[order_key])
+                    else:
+                        values.append(
+                            field_path_module.get_nested_value(order_key, data)
+                        )
                 except KeyError:
                     msg = _MISSING_ORDER_BY.format(order_key, data)
                     raise ValueError(msg)


Test patch:
diff --git a/firestore/tests/system/test_system.py b/firestore/tests/system/test_system.py
--- a/firestore/tests/system/test_system.py
+++ b/firestore/tests/system/test_system.py
@@ -611,6 +611,43 @@ def test_query_stream(client, cleanup):
         assert value["b"] == 2
 
 
+def test_query_with_order_dot_key(client, cleanup):
+    db = client
+    collection_id = "collek" + unique_resource_id("-")
+    collection = db.collection(collection_id)
+    for index in range(100, -1, -1):
+        doc = collection.document("test_{:09d}".format(index))
+        data = {"count": 10 * index, "wordcount": {"page1": index * 10 + 100}}
+        doc.set(data)
+        cleanup(doc.delete)
+    query = collection.order_by("wordcount.page1").limit(3)
+    data = [doc.to_dict()["wordcount"]["page1"] for doc in query.stream()]
+    assert [100, 110, 120] == data
+    for snapshot in collection.order_by("wordcount.page1").limit(3).stream():
+        last_value = snapshot.get("wordcount.page1")
+    cursor_with_nested_keys = {"wordcount": {"page1": last_value}}
+    found = list(
+        collection.order_by("wordcount.page1")
+        .start_after(cursor_with_nested_keys)
+        .limit(3)
+        .stream()
+    )
+    found_data = [
+        {u"count": 30, u"wordcount": {u"page1": 130}},
+        {u"count": 40, u"wordcount": {u"page1": 140}},
+        {u"count": 50, u"wordcount": {u"page1": 150}},
+    ]
+    assert found_data == [snap.to_dict() for snap in found]
+    cursor_with_dotted_paths = {"wordcount.page1": last_value}
+    cursor_with_key_data = list(
+        collection.order_by("wordcount.page1")
+        .start_after(cursor_with_dotted_paths)
+        .limit(3)
+        .stream()
+    )
+    assert found_data == [snap.to_dict() for snap in cursor_with_key_data]
+
+
 def test_query_unary(client, cleanup):
     collection_name = "unary" + UNIQUE_RESOURCE_ID
     collection = client.collection(collection_name)
diff --git a/firestore/tests/unit/v1/test_query.py b/firestore/tests/unit/v1/test_query.py
--- a/firestore/tests/unit/v1/test_query.py
+++ b/firestore/tests/unit/v1/test_query.py
@@ -808,6 +808,16 @@ def test__normalize_cursor_as_dict_hit(self):
 
         self.assertEqual(query._normalize_cursor(cursor, query._orders), ([1], True))
 
+    def test__normalize_cursor_as_dict_with_dot_key_hit(self):
+        cursor = ({"b.a": 1}, True)
+        query = self._make_one(mock.sentinel.parent).order_by("b.a", "ASCENDING")
+        self.assertEqual(query._normalize_cursor(cursor, query._orders), ([1], True))
+
+    def test__normalize_cursor_as_dict_with_inner_data_hit(self):
+        cursor = ({"b": {"a": 1}}, True)
+        query = self._make_one(mock.sentinel.parent).order_by("b.a", "ASCENDING")
+        self.assertEqual(query._normalize_cursor(cursor, query._orders), ([1], True))
+
     def test__normalize_cursor_as_snapshot_hit(self):
         values = {"b": 1}
         docref = self._make_docref("here", "doc_id")