Repository: Kinto/kinto

Issue:
Cannot select multiple fields in sub tree: parent.subparent.field
We noticed a problem with the 'fields' filtering in recently deployed versions.

Works: Just show attachment.original.filename field
https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records?_fields=attachment.original.filename

Works: Just show the attachment.original.filename.hash field
https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records?_fields=attachment.original.hash

Does **NOT** work: Just show the attachment.original.filename and attachment.original.filename.hash fields (Only returns hash):
https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records?_fields=attachment.original.filename,attachment.original.hash

Does work: Just show the attachment.original.filename and attachment.location fields:
https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records?_fields=attachment.original.filename,attachment.location

It seems like as soon as we select more than one field in attachment.original.* just one field is returned.



Discussion:
Thanks for reporting!

I confirm :|

```
$ echo '{"data": {"a": {"b": {"c1":1, "c2":2}}}}' | http :8888/v1/buckets/default/collections/test/records --auth user:pass
HTTP/1.1 201 Created
Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff
Content-Length: 213
Content-Type: application/json; charset=UTF-8
Date: Thu, 17 Nov 2016 10:11:26 GMT
Server: waitress

{
    "data": {
        "a": {
            "b": {
                "c1": 1, 
                "c2": 2
            }
        }, 
        "id": "26ca2cc8-1cff-4d73-88a8-5cde5da05546", 
        "last_modified": 1479377486159
    }, 
    "permissions": {
        "write": [
            "basicauth:cbd3731f18c97ebe1d31d9846b5f1b95cf8eeeae586e201277263434041e99d1"
        ]
    }
}


```

```
$ http :8888/v1/buckets/default/collections/test/records?_fields=a.b.c1,a.b.c2 --auth user:pass
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Content-Length, Expires, Alert, Retry-After, Last-Modified, Total-Records, ETag, Pragma, Cache-Control, Backoff, Next-Page
Cache-Control: no-cache, no-store
Content-Length: 105
Content-Type: application/json; charset=UTF-8
Date: Thu, 17 Nov 2016 10:13:01 GMT
Etag: "1479377486159"
Last-Modified: Thu, 17 Nov 2016 10:11:26 GMT
Server: waitress
Total-Records: 1

{
    "data": [
        {
            "a": {
                "b": {
                    "c2": 2
                }
            }, 
            "id": "26ca2cc8-1cff-4d73-88a8-5cde5da05546", 
            "last_modified": 1479377486159
        }
    ]
}
```

But do you think the behaviour changed in recent versions? How critical is it?


Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -74,7 +74,7 @@
 # The short X.Y version.
 version = '4.3'
 # The full version, including alpha/beta/rc tags.
-release = '4.3.3'
+release = '4.3.4'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
diff --git a/kinto/core/utils.py b/kinto/core/utils.py
--- a/kinto/core/utils.py
+++ b/kinto/core/utils.py
@@ -1,4 +1,5 @@
 import ast
+import collections
 import hashlib
 import hmac
 import os
@@ -175,9 +176,9 @@ def dict_subset(d, keys):
     for key in keys:
         if '.' in key:
             field, subfield = key.split('.', 1)
-            if isinstance(d.get(field), dict):
+            if isinstance(d.get(field), collections.Mapping):
                 subvalue = dict_subset(d[field], [subfield])
-                result.setdefault(field, {}).update(subvalue)
+                result[field] = dict_merge(subvalue, result.get(field, {}))
             elif field in d:
                 result[field] = d[field]
         else:
@@ -187,6 +188,16 @@ def dict_subset(d, keys):
     return result
 
 
+def dict_merge(a, b):
+    """Merge the two specified dicts"""
+    result = dict(**b)
+    for key, value in a.items():
+        if isinstance(value, collections.Mapping):
+            value = dict_merge(value, result.setdefault(key, {}))
+        result[key] = value
+    return result
+
+
 class COMPARISON(Enum):
     LT = '<'
     MIN = '>='
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -84,7 +84,7 @@ def read_file(filename):
 
 
 setup(name='kinto',
-      version='4.3.4.dev0',
+      version='4.3.4',
       description='Kinto Web Service - Store, Sync, Share, and Self-Host.',
       long_description=README + "\n\n" + CHANGELOG + "\n\n" + CONTRIBUTORS,
       license='Apache License (2.0)',


Test patch:
diff --git a/tests/core/resource/test_partial_response.py b/tests/core/resource/test_partial_response.py
--- a/tests/core/resource/test_partial_response.py
+++ b/tests/core/resource/test_partial_response.py
@@ -19,6 +19,7 @@ def setUp(self):
                     'nested': {
                         'size': 12546,
                         'hash': '0x1254',
+                        'mime': 'image/png',
                     }
                 }
             })
@@ -26,7 +27,7 @@ def setUp(self):
         self.resource.request = self.get_request()
 
 
-class BasicTest(PartialResponseBase):
+class PartialFieldsTest(PartialResponseBase):
 
     def test_fields_parameter_do_projection_on_get(self):
         self.resource.request.GET['_fields'] = 'field'
@@ -73,6 +74,14 @@ def test_nested_parameter_can_be_filtered_on_multiple_levels(self):
         self.assertIn('nested', record['data']['orig'])
         self.assertIn('size', record['data']['orig']['nested'])
         self.assertNotIn('hash', record['data']['orig']['nested'])
+        self.assertNotIn('mime', record['data']['orig']['nested'])
+
+    def test_can_filter_on_several_nested_fields(self):
+        self.resource.request.GET['_fields'] = 'orig.nested.size,orig.nested.hash'
+        record = self.resource.get()
+        self.assertIn('size', record['data']['orig']['nested'])
+        self.assertIn('hash', record['data']['orig']['nested'])
+        self.assertNotIn('mime', record['data']['orig']['nested'])
 
 
 class PermissionTest(PartialResponseBase):
diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py
--- a/tests/core/test_utils.py
+++ b/tests/core/test_utils.py
@@ -15,7 +15,7 @@
 from kinto.core.utils import (
     native_value, strip_whitespace, random_bytes_hex, read_env, hmac_digest,
     current_service, encode_header, decode_header, follow_subrequest,
-    build_request, dict_subset
+    build_request, dict_subset, dict_merge
 )
 from kinto.core.testing import DummyRequest
 
@@ -239,9 +239,9 @@ def test_can_filter_subobjects_keys(self):
         self.assertEqual(obtained, expected)
 
     def test_can_filter_subobjects_recursively(self):
-        input = dict(a=1, b=dict(c=2, d=dict(e=4, f=5)))
-        obtained = dict_subset(input, ["a", "b.d.e"])
-        expected = dict(a=1, b=dict(d=dict(e=4)))
+        input = dict(b=dict(c=2, d=dict(e=4, f=5, g=6)))
+        obtained = dict_subset(input, ["b.d.e", "b.d.f"])
+        expected = dict(b=dict(d=dict(e=4, f=5)))
         self.assertEqual(obtained, expected)
 
     def test_ignores_if_subobject_is_not_dict(self):
@@ -249,3 +249,11 @@ def test_ignores_if_subobject_is_not_dict(self):
         obtained = dict_subset(input, ["a", "b.c.d", "b.d"])
         expected = dict(a=1, b=dict(c=2, d=3))
         self.assertEqual(obtained, expected)
+
+
+class DictMergeTest(unittest.TestCase):
+
+    def test_merge(self):
+        obtained = dict_merge(dict(a=1, b=dict(c=2)), dict(b=dict(d=4)))
+        expected = dict(a=1, b=dict(c=2, d=4))
+        self.assertEqual(obtained, expected)