Repository: Kinto/kinto

Issue:
Filter/Sort on subobject properties
Let's say I have the following records:

```
{
    "data": {
        "title": "My title",
        "tags": ["astronomie", "science", "burundi"],
        "author": {
            "username": "natim",
            "first_name": "Rémy"
        }
    }
}
```

**How can I get all records from natim?**

http://localhost:8000/v1/buckets/wiki/collections/articles/records?author.username=natim ?



Discussion:
I would suppose the "." could be a delimiter.

I also think `.` is a good idea for that :+1: 
- `author.username=natim`

We use the dot notation for partial responses (`_fields=a.b`) so it seems the best option. 


Resolving patch:
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -956,7 +956,7 @@ def _extract_filters(self, queryparams=None):
                 )
                 continue
 
-            m = re.match(r'^(min|max|not|lt|gt|in|exclude)_(\w+)$', param)
+            m = re.match(r'^(min|max|not|lt|gt|in|exclude)_([\w\.]+)$', param)
             if m:
                 keyword, field = m.groups()
                 operator = getattr(COMPARISON, keyword.upper())
@@ -994,7 +994,7 @@ def _extract_sorting(self, limit):
         modified_field_used = self.model.modified_field in specified
         for field in specified:
             field = field.strip()
-            m = re.match(r'^([\-+]?)(\w+)$', field)
+            m = re.match(r'^([\-+]?)([\w\.]+)$', field)
             if m:
                 order, field = m.groups()
 
diff --git a/kinto/core/storage/memory.py b/kinto/core/storage/memory.py
--- a/kinto/core/storage/memory.py
+++ b/kinto/core/storage/memory.py
@@ -86,11 +86,16 @@ def apply_filters(self, records, filters):
         for record in records:
             matches = True
             for f in filters:
-                left = record.get(f.field)
                 right = f.value
+                left = record
+                subfields = f.field.split('.')
+                for subfield in subfields:
+                    if not isinstance(left, dict):
+                        break
+                    left = left.get(subfield)
+
                 if f.operator in (COMPARISON.IN, COMPARISON.EXCLUDE):
-                    right = left
-                    left = f.value
+                    right, left = left, right
                 else:
                     # Python3 cannot compare None to other value.
                     if left is None:
@@ -384,7 +389,13 @@ def apply_sorting(records, sorting):
 
     def column(first, record, name):
         empty = first.get(name, float('inf'))
-        return record.get(name, empty)
+        subfields = name.split('.')
+        value = record
+        for subfield in subfields:
+            value = value.get(subfield, empty)
+            if not isinstance(value, dict):
+                break
+        return value
 
     for sort in reversed(sorting):
         result = sorted(result,
diff --git a/kinto/core/storage/postgresql/__init__.py b/kinto/core/storage/postgresql/__init__.py
--- a/kinto/core/storage/postgresql/__init__.py
+++ b/kinto/core/storage/postgresql/__init__.py
@@ -624,16 +624,23 @@ def _format_conditions(self, filters, id_field, modified_field,
             elif filtr.field == modified_field:
                 sql_field = 'as_epoch(last_modified)'
             else:
-                # Safely escape field name
-                field_holder = '%s_field_%s' % (prefix, i)
-                holders[field_holder] = filtr.field
+                sql_field = "data"
+                # Subfields: ``person.name`` becomes ``data->person->>name``
+                subfields = filtr.field.split('.')
+                for j, subfield in enumerate(subfields):
+                    # Safely escape field name
+                    field_holder = '%s_field_%s_%s' % (prefix, i, j)
+                    holders[field_holder] = subfield
+                    # Use ->> to convert the last level to text.
+                    sql_field += "->>" if j == len(subfields) - 1 else "->"
+                    sql_field += ":%s" % field_holder
 
-                # JSON operator ->> retrieves values as text.
                 # If field is missing, we default to ''.
-                sql_field = "coalesce(data->>:%s, '')" % field_holder
+                sql_field = "coalesce(%s, '')" % sql_field
+                # Cast when comparing to number (eg. '4' < '12')
                 if isinstance(value, (int, float)) and \
                    value not in (True, False):
-                    sql_field = "(data->>:%s)::numeric" % field_holder
+                    sql_field += "::numeric"
 
             if filtr.operator not in (COMPARISON.IN, COMPARISON.EXCLUDE):
                 # For the IN operator, let psycopg escape the values list.
@@ -709,9 +716,15 @@ def _format_sorting(self, sorting, id_field, modified_field):
             elif sort.field == modified_field:
                 sql_field = 'last_modified'
             else:
-                field_holder = 'sort_field_%s' % i
-                holders[field_holder] = sort.field
-                sql_field = 'data->(:%s)' % field_holder
+                # Subfields: ``person.name`` becomes ``data->person->>name``
+                subfields = sort.field.split('.')
+                sql_field = 'data'
+                for j, subfield in enumerate(subfields):
+                    # Safely escape field name
+                    field_holder = 'sort_field_%s_%s' % (i, j)
+                    holders[field_holder] = subfield
+                    # Use ->> to convert the last level to text.
+                    sql_field += '->(:%s)' % field_holder
 
             sql_direction = 'ASC' if sort.direction > 0 else 'DESC'
             sql_sort = "%s %s" % (sql_field, sql_direction)


Test patch:
diff --git a/kinto/tests/core/resource/test_filter.py b/kinto/tests/core/resource/test_filter.py
--- a/kinto/tests/core/resource/test_filter.py
+++ b/kinto/tests/core/resource/test_filter.py
@@ -168,3 +168,32 @@ def test_exclude_returns_400_if_value_has_wrong_type(self):
         self.resource.request.GET = {'exclude_last_modified': 'a,b'}
         self.assertRaises(httpexceptions.HTTPBadRequest,
                           self.resource.collection_get)
+
+
+class SubobjectFilteringTest(BaseTest):
+    def setUp(self):
+        super(SubobjectFilteringTest, self).setUp()
+        self.patch_known_field.start()
+        for i in range(6):
+            record = {
+                'party': {'candidate': 'Marie', 'voters': i},
+                'location': 'Creuse'
+            }
+            self.model.create_record(record)
+
+    def test_records_can_be_filtered_by_subobjects(self):
+        self.resource.request.GET = {'party.voters': '1'}
+        result = self.resource.collection_get()
+        values = [item['party']['voters'] for item in result['data']]
+        self.assertEqual(sorted(values), [1])
+
+    def test_subobjects_filters_are_ignored_if_not_object(self):
+        self.resource.request.GET = {'location.city': 'barcelona'}
+        result = self.resource.collection_get()
+        self.assertEqual(len(result['data']), 0)
+
+    def test_subobjects_filters_works_with_directives(self):
+        self.resource.request.GET = {'in_party.voters': '1,2,3'}
+        result = self.resource.collection_get()
+        values = [item['party']['voters'] for item in result['data']]
+        self.assertEqual(sorted(values), [1, 2, 3])
diff --git a/kinto/tests/core/resource/test_sort.py b/kinto/tests/core/resource/test_sort.py
--- a/kinto/tests/core/resource/test_sort.py
+++ b/kinto/tests/core/resource/test_sort.py
@@ -114,3 +114,31 @@ def test_default_sort_is_last_modified_records_have_same_status(self):
         self.assertEqual(result['data'][1]['status'], 0)
         self.assertGreater(result['data'][0]['last_modified'],
                            result['data'][1]['last_modified'])
+
+
+class SubobjectSortingTest(BaseTest):
+    def setUp(self):
+        super(SubobjectSortingTest, self).setUp()
+        self.patch_known_field.start()
+
+        indices = list(range(20))
+        random.shuffle(indices)
+
+        for i in indices:
+            record = {
+                'party': {'candidate': 'Marie', 'voters': i},
+                'location': 'Creuse'
+            }
+            self.model.create_record(record)
+
+    def test_records_can_be_sorted_by_subobjects(self):
+        self.resource.request.GET = {'_sort': 'party.voters'}
+        result = self.resource.collection_get()
+        values = [item['party']['voters'] for item in result['data']]
+        self.assertEqual(sorted(values), values)
+
+    def test_subobjects_sorting_is_ignored_if_not_object(self):
+        self.resource.request.GET = {'_sort': 'location.city'}
+        result = self.resource.collection_get()
+        values = [item['party']['voters'] for item in result['data']]
+        self.assertNotEqual(sorted(values), values)
diff --git a/kinto/tests/core/test_storage.py b/kinto/tests/core/test_storage.py
--- a/kinto/tests/core/test_storage.py
+++ b/kinto/tests/core/test_storage.py
@@ -373,6 +373,17 @@ def test_get_all_handle_sorting_on_id(self):
                                           **self.storage_kw)
         self.assertTrue(records[0]['id'] < records[-1]['id'])
 
+    def test_get_all_handle_sorting_on_subobject(self):
+        for x in range(10):
+            record = dict(**self.record)
+            record["person"] = dict(age=x)
+            self.create_record(record)
+        sorting = [Sort('person.age', 1)]
+        records, _ = self.storage.get_all(sorting=sorting,
+                                          **self.storage_kw)
+        self.assertLess(records[0]['person']['age'],
+                        records[-1]['person']['age'])
+
     def test_get_all_can_filter_with_list_of_values(self):
         for l in ['a', 'b', 'c']:
             self.create_record({'code': l})
@@ -457,6 +468,14 @@ def test_get_all_can_filter_with_list_of_excluded_values(self):
                                           **self.storage_kw)
         self.assertEqual(len(records), 1)
 
+    def test_get_all_can_filter_by_subobjects_values(self):
+        for l in ['a', 'b', 'c']:
+            self.create_record({'code': {'sub': l}})
+        filters = [Filter('code.sub', 'a', utils.COMPARISON.EQ)]
+        records, _ = self.storage.get_all(filters=filters,
+                                          **self.storage_kw)
+        self.assertEqual(len(records), 1)
+
     def test_get_all_handle_a_pagination_rules(self):
         for x in range(10):
             record = dict(self.record)