Repository: Kinto/kinto

Issue:
Setting value in cache should never fail
The cache is a key value store. The transaction isolation and integrity constraints are details of implementation. Setting a value in the cache should just never fail.
Setting value in cache should never fail
The cache is a key value store. The transaction isolation and integrity constraints are details of implementation. Setting a value in the cache should just never fail.


Discussion:

Resolving patch:
diff --git a/kinto/core/cache/postgresql/__init__.py b/kinto/core/cache/postgresql/__init__.py
--- a/kinto/core/cache/postgresql/__init__.py
+++ b/kinto/core/cache/postgresql/__init__.py
@@ -1,13 +1,41 @@
 from __future__ import absolute_import
+from functools import wraps
 
 import os
+import time
 
 from kinto.core import logger
 from kinto.core.cache import CacheBase
 from kinto.core.storage.postgresql.client import create_from_config
+from kinto.core.storage.exceptions import BackendError
 from kinto.core.utils import json
 
 
+DELAY_BETWEEN_RETRIES_IN_SECONDS = 0.005
+MAX_RETRIES = 10
+
+
+def retry_on_failure(func):
+    try:
+        import psycopg2
+    except ImportError:  # pragma: no cover
+        pass  # Do not break (but will fail nicely later anyway)
+
+    @wraps(func)
+    def wraps_func(self, *args, **kwargs):
+        tries = kwargs.pop('tries', 0)
+        try:
+            return func(self, *args, **kwargs)
+        except psycopg2.IntegrityError as e:
+            if tries < MAX_RETRIES:
+                # Skip delay the 2 first times.
+                delay = max(0, tries - 1) * DELAY_BETWEEN_RETRIES_IN_SECONDS
+                time.sleep(delay)
+                return wraps_func(self, tries=(tries + 1), *args, **kwargs)
+            raise BackendError(original=e)
+    return wraps_func
+
+
 class Cache(CacheBase):
     """Cache backend using PostgreSQL.
 
@@ -121,6 +149,7 @@ def expire(self, key, ttl):
         with self.client.connect() as conn:
             conn.execute(query, dict(ttl=ttl, key=self.prefix + key))
 
+    @retry_on_failure
     def set(self, key, value, ttl=None):
         if ttl is None:
             logger.warning("No TTL for cache key %r" % key)


Test patch:
diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py
--- a/tests/core/test_cache.py
+++ b/tests/core/test_cache.py
@@ -5,9 +5,9 @@
 from kinto.core.utils import sqlalchemy
 from kinto.core.cache import (CacheBase, memory as memory_backend,
                               postgresql as postgresql_backend)
-
-from kinto.core.testing import skip_if_no_postgresql
 from kinto.core.cache.testing import CacheTest
+from kinto.core.storage.exceptions import BackendError
+from kinto.core.testing import skip_if_no_postgresql
 
 
 class CacheBaseTest(unittest.TestCase):
@@ -94,3 +94,19 @@ def setUp(self):
             self.cache.client,
             'session_factory',
             side_effect=sqlalchemy.exc.SQLAlchemyError)
+
+    def test_retry_on_integrity_error(self):
+        import psycopg2
+        connect = mock.MagicMock()
+        connect.__enter__.return_value.execute.side_effect = [psycopg2.IntegrityError, None]
+        with mock.patch.object(self.cache.client, 'connect', return_value=connect):
+            self.cache.set('this[]', 'value')
+            assert connect.__enter__.return_value.execute.call_count == 2
+
+    def test_retry_on_integrity_error_and_raises_after_a_few_tries(self):
+        import psycopg2
+        connect = mock.MagicMock()
+        connect.__enter__.return_value.execute.side_effect = psycopg2.IntegrityError
+        with mock.patch.object(self.cache.client, 'connect', return_value=connect):
+            with self.assertRaises(BackendError):
+                self.cache.set('this[]', 'value')