Repository: Kinto/kinto

Issue:
Should the cache backend even use transactions?
mozilla-services/kinto-fxa#33 seems to be related to the fact that we're opening a transaction to consult the authentication cache. Is this really necessary? It seems like for a cache, we don't need any of the ACID properties. /cc @leplatrem 

Should the cache backend even use transactions?
mozilla-services/kinto-fxa#33 seems to be related to the fact that we're opening a transaction to consult the authentication cache. Is this really necessary? It seems like for a cache, we don't need any of the ACID properties. /cc @leplatrem 



Discussion:
AFAIK there's always a transaction (isolation) for every operation. Nevertheless, a transaction does not necessary last more than one instruction.

So yes, I agree the cache operations could be decorrelated from the transactions managed (ie. opened/committed/rolledback) along the request cycle!

As @Natim noted, it would also allow to use the cache in `AfterResourceChanged` hooks, that are executed once the request transaction is committed.

AFAIK there's always a transaction (isolation) for every operation. Nevertheless, a transaction does not necessary last more than one instruction.

So yes, I agree the cache operations could be decorrelated from the transactions managed (ie. opened/committed/rolledback) along the request cycle!

As @Natim noted, it would also allow to use the cache in `AfterResourceChanged` hooks, that are executed once the request transaction is committed.


Resolving patch:
diff --git a/kinto/core/cache/postgresql/__init__.py b/kinto/core/cache/postgresql/__init__.py
--- a/kinto/core/cache/postgresql/__init__.py
+++ b/kinto/core/cache/postgresql/__init__.py
@@ -156,5 +156,5 @@ def delete(self, key):
 
 def load_from_config(config):
     settings = config.get_settings()
-    client = create_from_config(config, prefix='cache_')
+    client = create_from_config(config, prefix='cache_', with_transaction=False)
     return Cache(client=client, cache_prefix=settings['cache_prefix'])
diff --git a/kinto/core/storage/postgresql/client.py b/kinto/core/storage/postgresql/client.py
--- a/kinto/core/storage/postgresql/client.py
+++ b/kinto/core/storage/postgresql/client.py
@@ -62,7 +62,7 @@ def connect(self, readonly=False, force_commit=False):
 _CLIENTS = defaultdict(dict)
 
 
-def create_from_config(config, prefix=''):
+def create_from_config(config, prefix='', with_transaction=True):
     """Create a PostgreSQLClient client using settings in the provided config.
     """
     if sqlalchemy is None:
@@ -77,8 +77,9 @@ def create_from_config(config, prefix=''):
     # Custom Kinto settings, unsupported by SQLAlchemy.
     settings.pop(prefix + 'backend', None)
     settings.pop(prefix + 'max_fetch_size', None)
+    settings.pop(prefix + 'max_size_bytes', None)
     settings.pop(prefix + 'prefix', None)
-    transaction_per_request = settings.pop('transaction_per_request', False)
+    transaction_per_request = with_transaction and settings.pop('transaction_per_request', False)
 
     url = settings[prefix + 'url']
     existing_client = _CLIENTS[transaction_per_request].get(url)


Test patch:
diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py
--- a/tests/core/test_cache.py
+++ b/tests/core/test_cache.py
@@ -82,6 +82,7 @@ def test_add_over_quota_clean_oversized_items(self):
 class PostgreSQLCacheTest(CacheTest, unittest.TestCase):
     backend = postgresql_backend
     settings = {
+        'cache_backend': 'kinto.core.cache.postgresql',
         'cache_pool_size': 10,
         'cache_url': 'postgres://postgres:postgres@localhost:5432/testdb',
         'cache_prefix': ''
diff --git a/tests/core/test_views_transaction.py b/tests/core/test_views_transaction.py
--- a/tests/core/test_views_transaction.py
+++ b/tests/core/test_views_transaction.py
@@ -13,24 +13,17 @@
 
 
 class PostgreSQLTest(BaseWebTest):
-    def setUp(self):
-        super(PostgreSQLTest, self).setUp()
-        self.storage.initialize_schema()
-        self.permission.initialize_schema()
-
-    def tearDown(self):
-        super(BaseWebTest, self).tearDown()
-        self.storage.flush()
-        self.permission.flush()
-
     def get_app_settings(self, extras=None):
         settings = super(PostgreSQLTest, self).get_app_settings(extras)
         if sqlalchemy is not None:
             from .test_storage import PostgreSQLStorageTest
+            from .test_cache import PostgreSQLCacheTest
             from .test_permission import PostgreSQLPermissionTest
             settings.update(**PostgreSQLStorageTest.settings)
+            settings.update(**PostgreSQLCacheTest.settings)
             settings.update(**PostgreSQLPermissionTest.settings)
             settings.pop('storage_poolclass', None)
+            settings.pop('cache_poolclass', None)
             settings.pop('permission_poolclass', None)
         return settings
 
@@ -129,6 +122,34 @@ def test_modifications_are_rolled_back_on_error_accross_backends(self):
         self.assertEqual(len(resp.json['data']), 0)
 
 
+@skip_if_no_postgresql
+class TransactionCacheTest(PostgreSQLTest, unittest.TestCase):
+    def setUp(self):
+        def cache_and_fails(this, *args, **kwargs):
+            self.cache.set('test-cache', 'a value', ttl=100)
+            raise BackendError('boom')
+
+        patch = mock.patch.object(
+            self.permission,
+            'add_principal_to_ace',
+            wraps=cache_and_fails)
+        self.addCleanup(patch.stop)
+        patch.start()
+
+    def test_cache_backend_operations_are_always_committed(self):
+        self.app.post_json('/psilos',
+                           {'data': {'name': 'Amanite'}},
+                           headers=self.headers,
+                           status=503)
+
+        # Storage was rolled back.
+        resp = self.app.get('/psilos', headers=self.headers)
+        self.assertEqual(len(resp.json['data']), 0)
+
+        # Cache was committed.
+        self.assertEqual(self.cache.get('test-cache'), 'a value')
+
+
 @skip_if_no_postgresql
 class TransactionEventsTest(PostgreSQLTest, unittest.TestCase):
     def make_app_with_subscribers(self, subscribers):