Repository: Kinto/kinto

Issue:
Merge PATCH doesn't work on permissions
```
$ http -a 'user:pass' http://localhost:8888/v1/buckets/b4/collections/cc
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Expires, Content-Length, Pragma, Backoff, Cache-Control, Alert, Retry-After, ETag, Last-Modified
Cache-Control: no-cache, no-store
Content-Length: 182
Content-Type: application/json
Date: Fri, 18 Aug 2017 16:52:04 GMT
Etag: "1503075082664"
Last-Modified: Fri, 18 Aug 2017 16:51:22 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "a": "whoo",
        "id": "cc",
        "last_modified": 1503075082664
    },
    "permissions": {
        "read": [
            "hi"
        ],
        "write": [
            "basicauth:08eee2542a2328390dfaa7390fa7df0d64a7081b44d629bfacd9b4e02bd2676d"
        ]
    }
```

Let's get rid of the `read` permissions.

```
$ http -a 'user:pass'  PATCH http://localhost:8888/v1/buckets/b4/collections/cc "Content-Type:application/merge-patch+json" data:='{"a": null}' permissions:='{"read": null}'
HTTP/1.1 400 Bad Request
Access-Control-Expose-Headers: Alert, Backoff, Content-Length, Retry-After
Content-Length: 215
Content-Type: application/json
Date: Fri, 18 Aug 2017 16:52:47 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "code": 400,
    "details": [
        {
            "description": "\"None\" is not iterable",
            "location": "body",
            "name": "permissions.read"
        }
    ],
    "errno": 107,
    "error": "Invalid parameters",
    "message": "permissions.read in body: \"None\" is not iterable"
}
```

Note that JSON Patch does work by applying changes uniformly to `data` and `permissions`.
Merge PATCH doesn't work on permissions
```
$ http -a 'user:pass' http://localhost:8888/v1/buckets/b4/collections/cc
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Expires, Content-Length, Pragma, Backoff, Cache-Control, Alert, Retry-After, ETag, Last-Modified
Cache-Control: no-cache, no-store
Content-Length: 182
Content-Type: application/json
Date: Fri, 18 Aug 2017 16:52:04 GMT
Etag: "1503075082664"
Last-Modified: Fri, 18 Aug 2017 16:51:22 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "a": "whoo",
        "id": "cc",
        "last_modified": 1503075082664
    },
    "permissions": {
        "read": [
            "hi"
        ],
        "write": [
            "basicauth:08eee2542a2328390dfaa7390fa7df0d64a7081b44d629bfacd9b4e02bd2676d"
        ]
    }
```

Let's get rid of the `read` permissions.

```
$ http -a 'user:pass'  PATCH http://localhost:8888/v1/buckets/b4/collections/cc "Content-Type:application/merge-patch+json" data:='{"a": null}' permissions:='{"read": null}'
HTTP/1.1 400 Bad Request
Access-Control-Expose-Headers: Alert, Backoff, Content-Length, Retry-After
Content-Length: 215
Content-Type: application/json
Date: Fri, 18 Aug 2017 16:52:47 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "code": 400,
    "details": [
        {
            "description": "\"None\" is not iterable",
            "location": "body",
            "name": "permissions.read"
        }
    ],
    "errno": 107,
    "error": "Invalid parameters",
    "message": "permissions.read in body: \"None\" is not iterable"
}
```

Note that JSON Patch does work by applying changes uniformly to `data` and `permissions`.


Discussion:
 Null read permissions is failing colander validation. This is easy to fix if we allow ``{"permissions": {"read": null}`` on ``PermissionsSchema``, but that poses another problem: should we allow permissions to be ``null`` on "regular" JSON? Which would be the behavior? Do nothing? Or should we add some custom validation to raise an exception?
related to https://github.com/Pylons/colander/issues/299
 Null read permissions is failing colander validation. This is easy to fix if we allow ``{"permissions": {"read": null}`` on ``PermissionsSchema``, but that poses another problem: should we allow permissions to be ``null`` on "regular" JSON? Which would be the behavior? Do nothing? Or should we add some custom validation to raise an exception?
related to https://github.com/Pylons/colander/issues/299

Resolving patch:
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -159,6 +159,7 @@ def __init__(self, request, context=None):
 
         content_type = str(self.request.headers.get('Content-Type')).lower()
         self._is_json_patch = content_type == 'application/json-patch+json'
+        self._is_merge_patch = content_type == 'application/merge-patch+json'
 
         # Models are isolated by user.
         parent_id = self.get_parent_id(request)
@@ -674,10 +675,9 @@ def apply_changes(self, record, requested_changes):
             applied_changes = {**requested_changes}
             updated = {**record}
 
-            content_type = str(self.request.headers.get('Content-Type')).lower()
             # recursive patch and remove field if null attribute is passed (RFC 7396)
-            if content_type == 'application/merge-patch+json':
-                recursive_update_dict(updated, applied_changes, ignores=[None])
+            if self._is_merge_patch:
+                recursive_update_dict(updated, applied_changes, ignores=(None,))
             else:
                 updated.update(**applied_changes)
 
@@ -1185,17 +1185,28 @@ def process_record(self, new, old=None):
         new = super().process_record(new, old)
 
         # patch is specified as a list of of operations (RFC 6902)
+
+        payload = self.request.validated['body']
+
         if self._is_json_patch:
-            changes = self.request.validated['body']
-            permissions = apply_json_patch(old, changes)['permissions']
+            permissions = apply_json_patch(old, payload)['permissions']
+
+        elif self._is_merge_patch:
+            existing = old or {}
+            permissions = existing.get('__permissions__', {})
+            recursive_update_dict(permissions,
+                                  payload.get('permissions', {}),
+                                  ignores=(None,))
+
         else:
-            permissions = self.request.validated['body'].get('permissions', {})
+            permissions = {k: v for k, v in payload.get('permissions', {}).items()
+                           if v is not None}
 
         annotated = {**new}
 
         if permissions:
             is_put = (self.request.method.lower() == 'put')
-            if is_put:
+            if is_put or self._is_merge_patch:
                 # Remove every existing ACEs using empty lists.
                 for perm in self.permissions:
                     permissions.setdefault(perm, [])
diff --git a/kinto/core/resource/schema.py b/kinto/core/resource/schema.py
--- a/kinto/core/resource/schema.py
+++ b/kinto/core/resource/schema.py
@@ -119,30 +119,44 @@ def schema_type(self):
             return colander.Mapping(unknown='preserve')
 
     def deserialize(self, cstruct=colander.null):
-
         # If permissions are not a mapping (e.g null or invalid), try deserializing
         if not isinstance(cstruct, dict):
             return super().deserialize(cstruct)
 
+        # If using application/merge-patch+json we need to allow null values as they
+        # represent removing a key.
+        cstruct, removed_keys = self._preprocess_null_perms(cstruct)
+
         # If permissions are listed, check fields and produce fancy error messages
         if self.known_perms:
             for perm in cstruct:
                 colander.OneOf(choices=self.known_perms)(self, perm)
-            return super().deserialize(cstruct)
+            permissions = super().deserialize(cstruct)
 
         # Else deserialize the fields that are not on the schema
-        permissions = {}
-        perm_schema = colander.SequenceSchema(colander.SchemaNode(colander.String()))
-        for perm, principals in cstruct.items():
-            permissions[perm] = perm_schema.deserialize(principals)
+        else:
+            permissions = {}
+            perm_schema = colander.SequenceSchema(colander.SchemaNode(colander.String()))
+            for perm, principals in cstruct.items():
+                permissions[perm] = perm_schema.deserialize(principals)
 
-        return permissions
+        return self._postprocess_null_perms(permissions, removed_keys)
 
     def _get_node_principals(self, perm):
         principal = colander.SchemaNode(colander.String())
         return colander.SchemaNode(colander.Sequence(), principal, name=perm,
                                    missing=colander.drop)
 
+    @staticmethod
+    def _preprocess_null_perms(cstruct):
+        keys = {k for k, v in cstruct.items() if v is None}
+        cleaned = {k: v for k, v in cstruct.items() if v is not None}
+        return cleaned, keys
+
+    @staticmethod
+    def _postprocess_null_perms(validated, keys):
+        validated.update({k: None for k in keys})
+        return validated
 
 # Header schemas
 


Test patch:
diff --git a/tests/core/resource/test_record.py b/tests/core/resource/test_record.py
--- a/tests/core/resource/test_record.py
+++ b/tests/core/resource/test_record.py
@@ -289,6 +289,7 @@ def setUp(self):
         self.resource.record_id = self.stored['id']
         self.headers = self.resource.request.headers
         self.headers['Content-Type'] = 'application/merge-patch+json'
+        self.resource._is_merge_patch = True
 
         class ArticleSchema(ResourceSchema):
             unread = colander.SchemaNode(colander.Boolean(), missing=colander.drop)
@@ -340,6 +341,7 @@ def test_merge_patch_doesnt_remove_attribute_if_false(self):
 
     def test_patch_doesnt_remove_attribute_if_not_merge_header(self):
         self.headers['Content-Type'] = 'application/json'
+        self.resource._is_merge_patch = False
         self.validated['body'] = {'data': {'field': 'aaa'}}
         self.resource.patch()
         self.validated['body'] = {'data': {'field': None}}
@@ -350,6 +352,7 @@ def test_patch_doesnt_remove_attribute_if_not_merge_header(self):
 
     def test_merge_patch_doesnt_remove_previously_inserted_nones(self):
         self.headers['Content-Type'] = 'application/json'
+        self.resource._is_merge_patch = False
         self.validated['body'] = {'data': {'field': 'aaa'}}
         result = self.resource.patch()['data']
         self.validated['body'] = {'data': {'field': None}}
diff --git a/tests/core/resource/test_schema.py b/tests/core/resource/test_schema.py
--- a/tests/core/resource/test_schema.py
+++ b/tests/core/resource/test_schema.py
@@ -78,6 +78,11 @@ def test_works_with_empty_list_of_principals(self):
         deserialized = self.schema.deserialize(perms)
         self.assertEqual(deserialized, perms)
 
+    def test_works_with_empty_none_permissions(self):
+        perms = {'can_cook': None}
+        deserialized = self.schema.deserialize(perms)
+        self.assertEqual(deserialized, perms)
+
     def test_raises_invalid_if_not_a_mapping(self):
         perms = ['gab']
         self.assertRaises(colander.Invalid,
diff --git a/tests/test_views_records.py b/tests/test_views_records.py
--- a/tests/test_views_records.py
+++ b/tests/test_views_records.py
@@ -326,6 +326,39 @@ def test_merge_patch_remove_nones(self):
                                    status=200)
         self.assertNotIn('one', resp.json['data']['grain'])
 
+    def test_merge_patch_permissions(self):
+        headers = {**self.headers, 'Content-Type': 'application/merge-patch+json'}
+
+        # Add perms
+        json = {'permissions': {'read': ['bad_guys'], 'write': ['good_guys']}}
+        resp = self.app.patch_json(self.record_url,
+                                   json,
+                                   headers=headers,
+                                   status=200)
+        result = resp.json['permissions']
+
+        self.assertIn('write', result)
+        self.assertIn('read', result)
+        self.assertIn('bad_guys', result['read'])
+        self.assertIn('good_guys', result['write'])
+
+        # Remove perms
+        resp = self.app.patch_json(self.record_url,
+                                   json,
+                                   headers=headers,
+                                   status=200)
+
+        json = {'permissions': {'read': None, 'write': ['good_guys']}}
+        resp = self.app.patch_json(self.record_url,
+                                   json,
+                                   headers=headers,
+                                   status=200)
+        result = resp.json['permissions']
+
+        self.assertIn('write', result)
+        self.assertNotIn('read', result)
+        self.assertIn('good_guys', result['write'])
+
 
 class RecordsViewPatchTest(BaseWebTest, unittest.TestCase):