Repository: Kinto/kinto

Issue:
Crash when concurrent delete/update operations occur
```
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid_tm\/__init__.py\", line 105, in tm_tween
    manager.commit()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_manager.py\", line 131, in commit
    return self.get().commit()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_transaction.py\", line 295, in commit
    self._callBeforeCommitHooks()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_transaction.py\", line 366, in _callBeforeCommitHooks
    hook(*args, **kws)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/core\/events.py\", line 76, in _notify_resource_events_before
    request.registry.notify(event)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid\/registry.py\", line 110, in notify
    [ _ for _ in self.subscribers(events, None) ]
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/zope\/interface\/registry.py\", line 442, in subscribers
    return self.adapters.subscribers(objects, provided)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/zope\/interface\/adapter.py\", line 598, in subscribers
    subscription(*objects)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid\/config\/adapters.py\", line 130, in subscriber_wrapper
    return derived_subscriber(*arg)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/statsd\/client.py\", line 30, in _wrapped
    return_value = f(*args, **kwargs)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/plugins\/quotas\/listener.py\", line 237, in on_resource_changed
    record=collection_info)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/statsd\/client.py\", line 30, in _wrapped
    return_value = f(*args, **kwargs)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/core\/storage\/postgresql\/__init__.py\", line 356, in update
    record[modified_field] = updated['last_modified']
TypeError: 'NoneType' object is not subscriptable"
```
Crash when concurrent delete/update operations occur
```
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid_tm\/__init__.py\", line 105, in tm_tween
    manager.commit()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_manager.py\", line 131, in commit
    return self.get().commit()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_transaction.py\", line 295, in commit
    self._callBeforeCommitHooks()
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/transaction\/_transaction.py\", line 366, in _callBeforeCommitHooks
    hook(*args, **kws)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/core\/events.py\", line 76, in _notify_resource_events_before
    request.registry.notify(event)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid\/registry.py\", line 110, in notify
    [ _ for _ in self.subscribers(events, None) ]
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/zope\/interface\/registry.py\", line 442, in subscribers
    return self.adapters.subscribers(objects, provided)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/zope\/interface\/adapter.py\", line 598, in subscribers
    subscription(*objects)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/pyramid\/config\/adapters.py\", line 130, in subscriber_wrapper
    return derived_subscriber(*arg)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/statsd\/client.py\", line 30, in _wrapped
    return_value = f(*args, **kwargs)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/plugins\/quotas\/listener.py\", line 237, in on_resource_changed
    record=collection_info)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/statsd\/client.py\", line 30, in _wrapped
    return_value = f(*args, **kwargs)
  File \"\/usr\/local\/lib\/python3.5\/site-packages\/kinto\/core\/storage\/postgresql\/__init__.py\", line 356, in update
    record[modified_field] = updated['last_modified']
TypeError: 'NoneType' object is not subscriptable"
```


Discussion:
Seems to occur quite often on prod
In `storage.update()` we first select the record to determine if we'll perform an update or an insert. 

If a deletion occurs concurrently, then — when the lock is released — the `UPDATE` query will not affect any row, and `updated = result.fetchone()` will return `None`
Seems to occur quite often on prod
In `storage.update()` we first select the record to determine if we'll perform an update or an insert. 

If a deletion occurs concurrently, then — when the lock is released — the `UPDATE` query will not affect any row, and `updated = result.fetchone()` will return `None`

Resolving patch:
diff --git a/kinto/core/storage/postgresql/__init__.py b/kinto/core/storage/postgresql/__init__.py
--- a/kinto/core/storage/postgresql/__init__.py
+++ b/kinto/core/storage/postgresql/__init__.py
@@ -312,7 +312,7 @@ def update(self, collection_id, parent_id, object_id, record,
         query_record.pop(id_field, None)
         query_record.pop(modified_field, None)
 
-        query_create = """
+        query = """
         WITH delete_potential_tombstone AS (
             DELETE FROM deleted
              WHERE id = :object_id
@@ -323,15 +323,10 @@ def update(self, collection_id, parent_id, object_id, record,
         VALUES (:object_id, :parent_id,
                 :collection_id, (:data)::JSONB,
                 from_epoch(:last_modified))
-        RETURNING as_epoch(last_modified) AS last_modified;
-        """
-
-        query_update = """
-        UPDATE records SET data=(:data)::JSONB,
-                           last_modified=from_epoch(:last_modified)
-        WHERE id = :object_id
-           AND parent_id = :parent_id
-           AND collection_id = :collection_id
+        ON CONFLICT (id, parent_id, collection_id) DO UPDATE
+            SET data=(:data)::JSONB,
+                last_modified = GREATEST(from_epoch(:last_modified),
+                                         EXCLUDED.last_modified)
         RETURNING as_epoch(last_modified) AS last_modified;
         """
         placeholders = dict(object_id=object_id,
@@ -340,22 +335,11 @@ def update(self, collection_id, parent_id, object_id, record,
                             last_modified=record.get(modified_field),
                             data=json.dumps(query_record))
 
-        record = {**record, id_field: object_id}
-
         with self.client.connect() as conn:
-            # Create or update ?
-            query = """
-            SELECT id FROM records
-            WHERE id = :object_id
-              AND parent_id = :parent_id
-              AND collection_id = :collection_id;
-            """
-            result = conn.execute(query, placeholders)
-            query = query_update if result.rowcount > 0 else query_create
-
             result = conn.execute(query, placeholders)
             updated = result.fetchone()
 
+        record = {**record, id_field: object_id}
         record[modified_field] = updated['last_modified']
         return record
 


Test patch:
diff --git a/kinto/core/storage/testing.py b/kinto/core/storage/testing.py
--- a/kinto/core/storage/testing.py
+++ b/kinto/core/storage/testing.py
@@ -677,13 +677,13 @@ def test_update_uses_specified_last_modified_if_in_future(self):
         # Check that record timestamp is the one specified.
         retrieved = self.storage.get(object_id=record_id, **self.storage_kw)
         self.assertGreater(retrieved[self.modified_field], timestamp_before)
-        self.assertEquals(retrieved[self.modified_field],
-                          stored[self.modified_field])
+        self.assertGreaterEqual(retrieved[self.modified_field],
+                                stored[self.modified_field])
 
         # Check that collection timestamp took the one specified (in future).
         timestamp = self.storage.collection_timestamp(**self.storage_kw)
         self.assertGreater(timestamp, timestamp_before)
-        self.assertEquals(timestamp, stored[self.modified_field])
+        self.assertEquals(timestamp, retrieved[self.modified_field])
 
     def test_update_ignores_specified_last_modified_if_in_the_past(self):
         stored = self.create_record()