Repository: Kinto/kinto

Issue:
Crash with RecordNotFound when deleting record in default bucket?
```
Exception(most recent call first)

RecordNotFoundError: 21e1d010-ae9a-4eb7-af95-0473b4b8867e
  File "kinto/core/utils.py", line 414, in follow_subrequest
    return request.invoke_subrequest(subrequest, **kwargs), subrequest
  File "pyramid/router.py", line 195, in invoke_subrequest
    return self.invoke_request(request, _use_tweens=use_tweens)
  File "pyramid/router.py", line 249, in invoke_request
    response = handle_request(request)
  File "pyramid/router.py", line 156, in handle_request
    view_name
  File "pyramid/view.py", line 642, in _call_view
    response = view_callable(context, request)
  File "pyramid/viewderivers.py", line 410, in viewresult_to_response
    result = view(context, request)
  File "pyramid/viewderivers.py", line 148, in _requestonly_view
    response = view(request)
  File "kinto/plugins/default_bucket/__init__.py", line 165, in default_bucket
    response = request.invoke_subrequest(subrequest)
  File "pyramid/router.py", line 195, in invoke_subrequest
    return self.invoke_request(request, _use_tweens=use_tweens)
  File "pyramid/router.py", line 249, in invoke_request
    response = handle_request(request)
  File "pyramid/router.py", line 156, in handle_request
    view_name
  File "pyramid/view.py", line 642, in _call_view
    response = view_callable(context, request)
  File "pyramid/config/views.py", line 181, in __call__
    return view(context, request)
  File "pyramid/viewderivers.py", line 390, in attr_view
    return view(context, request)
  File "pyramid/viewderivers.py", line 368, in predicate_wrapper
    return view(context, request)
  File "pyramid/viewderivers.py", line 301, in secured_view
    return view(context, request)
  File "pyramid/viewderivers.py", line 439, in rendered_view
    result = view(context, request)
  File "pyramid/viewderivers.py", line 148, in _requestonly_view
    response = view(request)
  File "cornice/service.py", line 494, in wrapper
    response = view_()
  File "kinto/core/resource/__init__.py", line 583, in delete
    deleted = self.model.delete_record(record, last_modified=last_modified)
  File "kinto/core/resource/model.py", line 344, in delete_record
    record_id, parent_id, last_modified=last_modified)
  File "kinto/core/resource/model.py", line 251, in delete_record
    last_modified=last_modified)
  File "statsd/client.py", line 37, in _wrapped
    return_value = f(*args, **kwargs)
  File "kinto/core/storage/postgresql/__init__.py", line 417, in delete
    raise exceptions.RecordNotFoundError(object_id)
```

request was `DELETE /v1/buckets/default/collections/notes/records/21e1d010-ae9a-4eb7-af95-0473b4b8867e`

it appears to be during a sync of kinto.js because there was this request before:

```

  | {"lang":"en-US,en;q=0.5","code":200,"errno":0,"querystring":{"exclude_id":"21e1d010-ae9a-4eb7-af95-0473b4b8867e","_sort":"-last_modified"},"agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:58.0) Gecko/20100101 Firefox/58.0","t":1,"time":"2018-03-28T12:47:28.122000","path":"/v1/buckets/default/collections/notes/records","method":"OPTIONS"}
-- | --
```

See https://sentry.prod.mozaws.net/operations/kinto-testpilot-prod/issues/3403504/
Crash with RecordNotFound when deleting record in default bucket?
```
Exception(most recent call first)

RecordNotFoundError: 21e1d010-ae9a-4eb7-af95-0473b4b8867e
  File "kinto/core/utils.py", line 414, in follow_subrequest
    return request.invoke_subrequest(subrequest, **kwargs), subrequest
  File "pyramid/router.py", line 195, in invoke_subrequest
    return self.invoke_request(request, _use_tweens=use_tweens)
  File "pyramid/router.py", line 249, in invoke_request
    response = handle_request(request)
  File "pyramid/router.py", line 156, in handle_request
    view_name
  File "pyramid/view.py", line 642, in _call_view
    response = view_callable(context, request)
  File "pyramid/viewderivers.py", line 410, in viewresult_to_response
    result = view(context, request)
  File "pyramid/viewderivers.py", line 148, in _requestonly_view
    response = view(request)
  File "kinto/plugins/default_bucket/__init__.py", line 165, in default_bucket
    response = request.invoke_subrequest(subrequest)
  File "pyramid/router.py", line 195, in invoke_subrequest
    return self.invoke_request(request, _use_tweens=use_tweens)
  File "pyramid/router.py", line 249, in invoke_request
    response = handle_request(request)
  File "pyramid/router.py", line 156, in handle_request
    view_name
  File "pyramid/view.py", line 642, in _call_view
    response = view_callable(context, request)
  File "pyramid/config/views.py", line 181, in __call__
    return view(context, request)
  File "pyramid/viewderivers.py", line 390, in attr_view
    return view(context, request)
  File "pyramid/viewderivers.py", line 368, in predicate_wrapper
    return view(context, request)
  File "pyramid/viewderivers.py", line 301, in secured_view
    return view(context, request)
  File "pyramid/viewderivers.py", line 439, in rendered_view
    result = view(context, request)
  File "pyramid/viewderivers.py", line 148, in _requestonly_view
    response = view(request)
  File "cornice/service.py", line 494, in wrapper
    response = view_()
  File "kinto/core/resource/__init__.py", line 583, in delete
    deleted = self.model.delete_record(record, last_modified=last_modified)
  File "kinto/core/resource/model.py", line 344, in delete_record
    record_id, parent_id, last_modified=last_modified)
  File "kinto/core/resource/model.py", line 251, in delete_record
    last_modified=last_modified)
  File "statsd/client.py", line 37, in _wrapped
    return_value = f(*args, **kwargs)
  File "kinto/core/storage/postgresql/__init__.py", line 417, in delete
    raise exceptions.RecordNotFoundError(object_id)
```

request was `DELETE /v1/buckets/default/collections/notes/records/21e1d010-ae9a-4eb7-af95-0473b4b8867e`

it appears to be during a sync of kinto.js because there was this request before:

```

  | {"lang":"en-US,en;q=0.5","code":200,"errno":0,"querystring":{"exclude_id":"21e1d010-ae9a-4eb7-af95-0473b4b8867e","_sort":"-last_modified"},"agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:58.0) Gecko/20100101 Firefox/58.0","t":1,"time":"2018-03-28T12:47:28.122000","path":"/v1/buckets/default/collections/notes/records","method":"OPTIONS"}
-- | --
```

See https://sentry.prod.mozaws.net/operations/kinto-testpilot-prod/issues/3403504/


Discussion:
More than likely from me 👋, but could not reproduce.
Using kinto.js, I was deleting an array of records using collection().delete(). On a loop of 8 may be, only one crashed, returning 500. I'm now using deleteAny (thought that was a good idea even if not so sure right now 🤔). Please let me know if you have questions or if I can help.
Thanks Sebastien for your help!

I don't know when we'll take the time to work on this though :sweat: 
@leplatrem could you check if you are getting more of these? Softvision is telling us they are getting a lot of those and all their DELETE ops are failing
cc @Natim 
You summoned me, you can now start with your first wish :couch_and_lamp: 
It should probably return a 404 instead of a 500
![screenshot from 2018-03-29 09-08-35](https://user-images.githubusercontent.com/546692/38099994-dfb14bda-3330-11e8-8de9-55da161ed29f.png)

> It should probably return a 404 instead of a 500

It looks like it's already returning 404
```
http PUT :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 201 Created
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 152
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:01 GMT
Etag: "1522339921768"
Last-Modified: Thu, 29 Mar 2018 16:12:01 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "id": "1",
        "last_modified": 1522339921768
    },
    "permissions": {
        "write": [
            "basicauth:7afa16e9e220bdca1f8fa1839c9cb7dc29cffbd148756908894ed8add8d1ced8"
        ]
    }
}

➜  kinto git:(master) ✗ http DELETE :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 64
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:07 GMT
Etag: "1522339927516"
Last-Modified: Thu, 29 Mar 2018 16:12:07 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "deleted": true,
        "id": "1",
        "last_modified": 1522339927516
    }
}

➜  kinto git:(master) ✗ http DELETE :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 404 Not Found
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 90
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:08 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "code": 404,
    "details": {
        "id": "1",
        "resource_name": "record"
    },
    "errno": 110,
    "error": "Not Found"
}
```
@leplatrem I haven't seen a 404 but we are seeing both: 

2 delete operations - 2 **200s**:


![conseq-delete](https://user-images.githubusercontent.com/128755/38100282-fa544d24-334a-11e8-9df3-92df1b380e0f.gif)

and then in another case: 2 delete operations - 1 **200**, 1 **500**:


![conseq-500](https://user-images.githubusercontent.com/128755/38100270-f0a1f1d2-334a-11e8-99c8-1bf07627bb09.gif)

Interesting...

I was playing with the batch endpoint too, but it seems fine.
```
$ echo '{"requests":[{"method":"DELETE","path":"/v1/buckets/default/collections/a/records/2"},{"method":"DELETE","path":"/v1/buckets/default/collections/a/records/2"}]}' | http POST :8888/v1/batch -a user:pass
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 758
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:16:23 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "responses": [
        {
            "body": {
                "data": {
                    "deleted": true,
                    "id": "2",
                    "last_modified": 1522340183528
                }
            },
            "headers": {
                "Access-Control-Expose-Headers": "Backoff, Retry-After, Alert, Content-Length",
                "Content-Length": "64",
                "Content-Type": "application/json",
                "ETag": "\"1522340183528\"",
                "Last-Modified": "Thu, 29 Mar 2018 16:16:23 GMT",
                "X-Content-Type-Options": "nosniff"
            },
            "path": "/v1/buckets/default/collections/a/records/2",
            "status": 200
        },
        {
            "body": {
                "code": 404,
                "details": {
                    "id": "2",
                    "resource_name": "record"
                },
                "errno": 110,
                "error": "Not Found"
            },
            "headers": {
                "Access-Control-Expose-Headers": "Backoff, Retry-After, Alert, Content-Length",
                "Content-Length": "90",
                "Content-Type": "application/json",
                "X-Content-Type-Options": "nosniff"
            },
            "path": "/v1/buckets/default/collections/a/records/2",
            "status": 404
        }
    ]
}
```

So you run the same delete request twice? And sometimes the second one fails. That might be related to #1407 

We should catch `ResourceNotFound` here:

https://github.com/Kinto/kinto/blob/c6cc7bba094aed6897d0157dc78b1731ac12c8db/kinto/core/resource/__init__.py#L573-L583

More than likely from me 👋, but could not reproduce.
Using kinto.js, I was deleting an array of records using collection().delete(). On a loop of 8 may be, only one crashed, returning 500. I'm now using deleteAny (thought that was a good idea even if not so sure right now 🤔). Please let me know if you have questions or if I can help.
Thanks Sebastien for your help!

I don't know when we'll take the time to work on this though :sweat: 
@leplatrem could you check if you are getting more of these? Softvision is telling us they are getting a lot of those and all their DELETE ops are failing
cc @Natim 
You summoned me, you can now start with your first wish :couch_and_lamp: 
It should probably return a 404 instead of a 500
![screenshot from 2018-03-29 09-08-35](https://user-images.githubusercontent.com/546692/38099994-dfb14bda-3330-11e8-8de9-55da161ed29f.png)

> It should probably return a 404 instead of a 500

It looks like it's already returning 404
```
http PUT :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 201 Created
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 152
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:01 GMT
Etag: "1522339921768"
Last-Modified: Thu, 29 Mar 2018 16:12:01 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "id": "1",
        "last_modified": 1522339921768
    },
    "permissions": {
        "write": [
            "basicauth:7afa16e9e220bdca1f8fa1839c9cb7dc29cffbd148756908894ed8add8d1ced8"
        ]
    }
}

➜  kinto git:(master) ✗ http DELETE :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 64
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:07 GMT
Etag: "1522339927516"
Last-Modified: Thu, 29 Mar 2018 16:12:07 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "data": {
        "deleted": true,
        "id": "1",
        "last_modified": 1522339927516
    }
}

➜  kinto git:(master) ✗ http DELETE :8888/v1/buckets/default/collections/a/records/1 -a user:pass
HTTP/1.1 404 Not Found
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 90
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:12:08 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "code": 404,
    "details": {
        "id": "1",
        "resource_name": "record"
    },
    "errno": 110,
    "error": "Not Found"
}
```
@leplatrem I haven't seen a 404 but we are seeing both: 

2 delete operations - 2 **200s**:


![conseq-delete](https://user-images.githubusercontent.com/128755/38100282-fa544d24-334a-11e8-9df3-92df1b380e0f.gif)

and then in another case: 2 delete operations - 1 **200**, 1 **500**:


![conseq-500](https://user-images.githubusercontent.com/128755/38100270-f0a1f1d2-334a-11e8-99c8-1bf07627bb09.gif)

Interesting...

I was playing with the batch endpoint too, but it seems fine.
```
$ echo '{"requests":[{"method":"DELETE","path":"/v1/buckets/default/collections/a/records/2"},{"method":"DELETE","path":"/v1/buckets/default/collections/a/records/2"}]}' | http POST :8888/v1/batch -a user:pass
HTTP/1.1 200 OK
Access-Control-Expose-Headers: Backoff, Retry-After, Alert, Content-Length
Content-Length: 758
Content-Type: application/json
Date: Thu, 29 Mar 2018 16:16:23 GMT
Server: waitress
X-Content-Type-Options: nosniff

{
    "responses": [
        {
            "body": {
                "data": {
                    "deleted": true,
                    "id": "2",
                    "last_modified": 1522340183528
                }
            },
            "headers": {
                "Access-Control-Expose-Headers": "Backoff, Retry-After, Alert, Content-Length",
                "Content-Length": "64",
                "Content-Type": "application/json",
                "ETag": "\"1522340183528\"",
                "Last-Modified": "Thu, 29 Mar 2018 16:16:23 GMT",
                "X-Content-Type-Options": "nosniff"
            },
            "path": "/v1/buckets/default/collections/a/records/2",
            "status": 200
        },
        {
            "body": {
                "code": 404,
                "details": {
                    "id": "2",
                    "resource_name": "record"
                },
                "errno": 110,
                "error": "Not Found"
            },
            "headers": {
                "Access-Control-Expose-Headers": "Backoff, Retry-After, Alert, Content-Length",
                "Content-Length": "90",
                "Content-Type": "application/json",
                "X-Content-Type-Options": "nosniff"
            },
            "path": "/v1/buckets/default/collections/a/records/2",
            "status": 404
        }
    ]
}
```

So you run the same delete request twice? And sometimes the second one fails. That might be related to #1407 

We should catch `ResourceNotFound` here:

https://github.com/Kinto/kinto/blob/c6cc7bba094aed6897d0157dc78b1731ac12c8db/kinto/core/resource/__init__.py#L573-L583


Resolving patch:
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -580,7 +580,25 @@ def delete(self):
         if last_modified and last_modified <= record[self.model.modified_field]:
             last_modified = None
 
-        deleted = self.model.delete_record(record, last_modified=last_modified)
+        try:
+            deleted = self.model.delete_record(record, last_modified=last_modified)
+        except storage_exceptions.RecordNotFoundError:
+            # Delete might fail if the object was deleted since we
+            # fetched it from the storage (ref Kinto/kinto#1407). This
+            # is one of a larger class of issues where another request
+            # could modify the record between our fetch and our
+            # delete, which could e.g. invalidate our precondition
+            # checking. Fixing this correctly is a larger
+            # problem. However, let's punt on fixing it correctly and
+            # just handle this one important case for now (see #1557).
+            #
+            # Raise a 404 vs. a 409 or 412 because that's what we
+            # would have done if the other thread's delete had
+            # happened a little earlier. (The client doesn't need to
+            # know that we did a bunch of work fetching the existing
+            # record for nothing.)
+            raise self._404_for_record(self.record_id)
+
         timestamp = deleted[self.model.modified_field]
         self._add_timestamp_header(self.request.response, timestamp=timestamp)
 
@@ -720,6 +738,14 @@ def postprocess(self, result, action=ACTIONS.READ, old=None):
     # Internals
     #
 
+    def _404_for_record(self, record_id):
+        details = {
+            'id': record_id,
+            'resource_name': self.request.current_resource_name
+        }
+        return http_error(HTTPNotFound(), errno=ERRORS.INVALID_RESOURCE_ID,
+                          details=details)
+
     def _get_record_or_404(self, record_id):
         """Retrieve record from storage and raise ``404 Not found`` if missing.
 
@@ -733,13 +759,7 @@ def _get_record_or_404(self, record_id):
         try:
             return self.model.get_record(record_id)
         except storage_exceptions.RecordNotFoundError:
-            details = {
-                'id': record_id,
-                'resource_name': self.request.current_resource_name
-            }
-            response = http_error(HTTPNotFound(), errno=ERRORS.INVALID_RESOURCE_ID,
-                                  details=details)
-            raise response
+            raise self._404_for_record(record_id)
 
     def _add_timestamp_header(self, response, timestamp=None):
         """Add current timestamp in response headers, when request comes in.


Test patch:
diff --git a/tests/core/resource/test_record.py b/tests/core/resource/test_record.py
--- a/tests/core/resource/test_record.py
+++ b/tests/core/resource/test_record.py
@@ -5,6 +5,7 @@
 
 from kinto.core.resource import ResourceSchema
 from kinto.core.errors import ERRORS
+from kinto.core.storage import exceptions as storage_exceptions
 
 from . import BaseTest
 
@@ -193,6 +194,15 @@ def test_delete_ignores_last_modified_if_less(self):
         result = self.resource.delete()['data']
         self.assertGreater(result[self.model.modified_field], last_modified)
 
+    def test_delete_returns_404_if_backend_throws(self):
+        def raise_recordnotfound(*args, **kwargs):
+            raise storage_exceptions.RecordNotFoundError
+
+        record = self.model.create_record({'field': 'value'})
+        self.resource.record_id = record['id']
+        with mock.patch.object(self.model, 'delete_record', side_effect=raise_recordnotfound):
+            self.assertRaises(httpexceptions.HTTPNotFound, self.resource.delete)
+
 
 class PatchTest(BaseTest):
     def setUp(self):