Repository: Kinto/kinto

Issue:
Error with paginated list of changes in the history endpoint
Setup, latest kinto master:

In `kinto.ini`, set `kinto.paginate_by` to `10` and enable `kinto.plugins.history`.

Then:

```http
http -a test: PUT :8888/v1/buckets/foo
http -a test: PUT :8888/v1/buckets/foo/collections/bar
echo '{"data":{"n":1}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":2}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":3}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":4}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":5}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":6}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":7}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":8}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":9}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":10}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":11}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
echo '{"data":{"n":12}}' | http -a test: POST :8888/v1/buckets/foo/collections/bar/records
```

Take the latest `last_modified` of the created records (the one with `n: 12`, eg. `1488187867432`), and execute this query (the idea it to get more than `10` change entries):

```http
http -a test: ":8888/v1/buckets/foo/history?collection_id=bar&max_target.data.last_modified=1488187867432&_sort=-target.data.last_modified&resource_name=record"
```

We get a 500 error and a nice Python stack trace:

```python
ERROR:kinto.core.views.errors:"GET   /v1/buckets/foo/history?resource_name=record&collection_id=bar&max_target.data.last_modified=1488187867432&_sort=-target.data.last_modified" ? (? ms) 'target.data.last_modified' agent=HTTPie/0.9.2 authn_type=basicauth collection_id=history collection_timestamp=1488187867433 errno=None exception=Traceback (most recent call last):
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/tweens.py", line 22, in excview_tween
    response = handler(request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 119, in tm_tween
    reraise(*exc_info)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid_tm/compat.py", line 15, in reraise
    raise value
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 98, in tm_tween
    response = handler(request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/router.py", line 158, in handle_request
    view_name
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/view.py", line 547, in _call_view
    response = view_callable(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/config/views.py", line 182, in __call__
    return view(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/viewderivers.py", line 393, in attr_view
    return view(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/viewderivers.py", line 371, in predicate_wrapper
    return view(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/viewderivers.py", line 302, in _secured_view
    return view(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/viewderivers.py", line 442, in rendered_view
    result = view(context, request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/pyramid/viewderivers.py", line 147, in _requestonly_view
    response = view(request)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/cornice/service.py", line 491, in wrapper
    response = view_()
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/kinto/core/resource/__init__.py", line 289, in collection_get
    next_page = self._next_page_url(sorting, limit, lastrecord, offset)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/kinto/core/resource/__init__.py", line 1060, in _next_page_url
    token = self._build_pagination_token(sorting, last_record, offset)
  File "/home/niko/work/kinto-http.js/.venv/lib/python3.5/site-packages/kinto/core/resource/__init__.py", line 1084, in _build_pagination_token
    token['last_record'][field] = last_record[field]
KeyError: 'target.data.last_modified' lang=None uid=35f724b781ae01543ed29fd50dc19ed3434594c9e97e40e97f136626c12a64ae
```


Discussion:
From the error it seems we're not splitting dotted values to retrieve the matching nested dict, hence the uncaught key error. 
Would something like this be enough?

```diff
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
index a4f9c89..237c6fc 100644
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -1087,7 +1087,7 @@ class UserResource:
         }
 
         for field, _ in sorting:
-            token['last_record'][field] = last_record[field]
+            token['last_record'][field] = dict_subset(last_record, [field])
 
         return encode64(json.dumps(token))
```

Edit: no, `dict_subset` doesn't do what we want it seems.

Resolving patch:
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -16,7 +16,7 @@
 from kinto.core.events import ACTIONS
 from kinto.core.storage import exceptions as storage_exceptions, Filter, Sort
 from kinto.core.utils import (
-    COMPARISON, classname, decode64, encode64, json,
+    COMPARISON, classname, decode64, encode64, json, find_nested_value,
     dict_subset, recursive_update_dict, apply_json_patch
 )
 
@@ -1079,7 +1079,9 @@ def _build_pagination_token(self, sorting, last_record, offset):
         }
 
         for field, _ in sorting:
-            token['last_record'][field] = last_record[field]
+            last_value = find_nested_value(last_record, field)
+            if last_value is not None:
+                token['last_record'][field] = last_value
 
         return encode64(json.dumps(token))
 
diff --git a/kinto/core/storage/memory.py b/kinto/core/storage/memory.py
--- a/kinto/core/storage/memory.py
+++ b/kinto/core/storage/memory.py
@@ -7,7 +7,7 @@
 from kinto.core.storage import (
     StorageBase, exceptions,
     DEFAULT_ID_FIELD, DEFAULT_MODIFIED_FIELD, DEFAULT_DELETED_FIELD)
-from kinto.core.utils import COMPARISON
+from kinto.core.utils import (COMPARISON, find_nested_value)
 
 
 def tree():
@@ -326,12 +326,7 @@ def apply_filters(records, filters):
                 if isinstance(right, int):
                     right = str(right)
 
-            left = record
-            subfields = f.field.split('.')
-            for subfield in subfields:
-                if not isinstance(left, dict):
-                    break
-                left = left.get(subfield)
+            left = find_nested_value(record, f.field)
 
             if f.operator in (COMPARISON.IN, COMPARISON.EXCLUDE):
                 right, left = left, right
@@ -362,14 +357,7 @@ def apply_sorting(records, sorting):
     first_record = result[0]
 
     def column(first, record, name):
-        empty = first.get(name, float('inf'))
-        subfields = name.split('.')  # Never empty.
-        value = record
-        for subfield in subfields:  # pragma: no branch
-            value = value.get(subfield, empty)
-            if not isinstance(value, dict):
-                break
-        return value
+        return find_nested_value(record, name, default=float('inf'))
 
     for sort in reversed(sorting):
         result = sorted(result,
diff --git a/kinto/core/utils.py b/kinto/core/utils.py
--- a/kinto/core/utils.py
+++ b/kinto/core/utils.py
@@ -182,6 +182,36 @@ def dict_merge(a, b):
     return result
 
 
+def find_nested_value(d, path, default=None):
+    """Finds a nested value in a dict from a dotted path key string.
+
+    :param dict d: the dict to retrieve nested value from
+    :param str path: the path to the nested value, in dot notation
+    :returns: the nested value if any was found, or None
+    """
+    if path in d:
+        return d.get(path)
+
+    # the challenge is to identify what is the root key, as dict keys may
+    # contain dot characters themselves
+    parts = path.split('.')
+
+    # build a list of all possible root keys from all the path parts
+    candidates = ['.'.join(parts[:i + 1]) for i in range(len(parts))]
+
+    # we start with the longest candidate paths as they're most likely to be the
+    # ones we want if they match
+    root = next((key for key in reversed(candidates) if key in d), None)
+
+    # if no valid root candidates were found, the path is invalid; abandon
+    if root is None or not isinstance(d.get(root), dict):
+        return default
+
+    # we have our root key, extract the new subpath and recur
+    subpath = path.replace(root + '.', '', 1)
+    return find_nested_value(d.get(root), subpath)
+
+
 class COMPARISON(Enum):
     LT = '<'
     MIN = '>='


Test patch:
diff --git a/tests/core/resource/test_pagination.py b/tests/core/resource/test_pagination.py
--- a/tests/core/resource/test_pagination.py
+++ b/tests/core/resource/test_pagination.py
@@ -235,7 +235,9 @@ def setUp(self):
         self.patch_known_field.start()
         self.record = {
             'id': 1, 'status': 2, 'unread': True,
-            'last_modified': 1234, 'title': 'Title'
+            'last_modified': 1234, 'title': 'Title',
+            'nested': {'subvalue': 42},
+            'nested.other': {'subvalue': 43},
         }
 
     def test_token_contains_current_offset(self):
@@ -281,3 +283,30 @@ def test_multiple_sorting_keep_all(self):
         self.assertEqual(tokeninfo['last_record'],
                          {"last_modified": 1234, "status": 2,
                           'title': 'Title'})
+
+    def test_sorting_on_nested_field(self):
+        token = self.resource._build_pagination_token([
+            ('nested.subvalue', -1),
+            ('title', 1),
+        ], self.record, 88)
+        tokeninfo = json.loads(b64decode(token).decode('ascii'))
+        self.assertEqual(tokeninfo['last_record'],
+                         {'title': 'Title', 'nested.subvalue': 42})
+
+    def test_disambiguate_fieldname_containing_dots(self):
+        token = self.resource._build_pagination_token([
+            ('nested.other.subvalue', -1),
+            ('title', 1),
+        ], self.record, 88)
+        tokeninfo = json.loads(b64decode(token).decode('ascii'))
+        self.assertEqual(tokeninfo['last_record'],
+                         {'title': 'Title', 'nested.other.subvalue': 43})
+
+    def test_strip_malformed_sort_field(self):
+        token = self.resource._build_pagination_token([
+            ('non.existent', -1),
+            ('title', 1),
+        ], self.record, 88)
+        tokeninfo = json.loads(b64decode(token).decode('ascii'))
+        self.assertEqual(tokeninfo['last_record'],
+                         {'title': 'Title'})
diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py
--- a/tests/core/test_utils.py
+++ b/tests/core/test_utils.py
@@ -13,7 +13,8 @@
 from kinto.core.utils import (
     native_value, strip_whitespace, random_bytes_hex, read_env, hmac_digest,
     current_service, follow_subrequest, build_request, dict_subset, dict_merge,
-    parse_resource, prefixed_principals, recursive_update_dict
+    parse_resource, prefixed_principals, recursive_update_dict,
+    find_nested_value
 )
 from kinto.core.testing import DummyRequest
 
@@ -227,6 +228,62 @@ def test_merge(self):
         self.assertEqual(obtained, expected)
 
 
+class FindNestedValueTest(unittest.TestCase):
+
+    def test_find_flat_value(self):
+        record = {"a": 42}
+        obtained = find_nested_value(record, "a")
+        self.assertEqual(obtained, 42)
+
+    def test_find_nested_value(self):
+        record = {"a": {"b": 42}}
+        obtained = find_nested_value(record, "a.b")
+        self.assertEqual(obtained, 42)
+
+    def test_find_deeply_nested_value(self):
+        record = {"a": {"b": {"c": 42}}}
+        obtained = find_nested_value(record, "a.b.c")
+        self.assertEqual(obtained, 42)
+
+    def test_find_dotted_path_value(self):
+        record = {"a.b": 42}
+        obtained = find_nested_value(record, "a.b")
+        self.assertEqual(obtained, 42)
+
+    def test_find_nested_dotted_path_value(self):
+        record = {"a": {"b.c": 42}}
+        obtained = find_nested_value(record, "a.b.c")
+        self.assertEqual(obtained, 42)
+
+        record = {"a.b": {"c.d": 42}}
+        obtained = find_nested_value(record, "a.b.c.d")
+        self.assertEqual(obtained, 42)
+
+        record = {"a": {"b": {"a": {"b": 42}}}}
+        obtained = find_nested_value(record, "a.b.a.b")
+        self.assertEqual(obtained, 42)
+
+    def test_find_disambiguated_dotted_path_values(self):
+        # XXX: To be honest, this is a really scary use case. Probably a
+        # limitation of the dotted path notation we may want to document.
+        # At least this test acts as documentation for now.
+        record = {"a": {"b": 0}, "a.b": 42}
+        obtained = find_nested_value(record, "a.b")
+        self.assertEqual(obtained, 42)
+
+    def test_unmatched_path_returns_none(self):
+        record = {"a": 42}
+        self.assertIsNone(find_nested_value(record, "x"))
+        self.assertIsNone(find_nested_value(record, "a.b"))
+        self.assertIsNone(find_nested_value(record, "x.a"))
+
+    def test_fallback_default_value(self):
+        record = {"a": 42}
+        self.assertEqual(find_nested_value(record, "x", 1337), 1337)
+        self.assertEqual(find_nested_value(record, "a.b", 1337), 1337)
+        self.assertEqual(find_nested_value(record, "x.a", 1337), 1337)
+
+
 class RecursiveUpdateDictTest(unittest.TestCase):
 
     def test_merge(self):