Repository: Kinto/kinto

Issue:
Write a memcache cache backend
Memcache is a great candidate for a cache backend, because it's a key-value cache with a TTL. One of our ops guys mentioned that memcache is more robust than redis because it's not master-slave replicated.


Discussion:

Resolving patch:
diff --git a/kinto/core/cache/memcached.py b/kinto/core/cache/memcached.py
new file mode 100644
--- /dev/null
+++ b/kinto/core/cache/memcached.py
@@ -0,0 +1,107 @@
+import logging
+from functools import wraps
+from math import ceil, floor
+from time import time
+
+from pyramid.settings import aslist
+
+from kinto.core.cache import CacheBase
+from kinto.core.storage import exceptions
+from kinto.core.utils import json, memcache
+
+logger = logging.getLogger(__name__)
+
+
+def wrap_memcached_error(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except TypeError:
+            raise
+        except (memcache.Client.MemcachedKeyError,
+                memcache.Client.MemcachedStringEncodingError) as e:
+            logger.exception(e)
+            raise exceptions.BackendError(original=e)
+    return wrapped
+
+
+def create_from_config(config, prefix=''):
+    """Redis client instantiation from settings.
+    """
+    settings = config.get_settings()
+    hosts = aslist(settings[prefix + 'hosts'])
+    return memcache.Client(hosts)
+
+
+class Cache(CacheBase):
+    """Cache backend implementation using Redis.
+
+    Enable in configuration::
+
+        kinto.cache_backend = kinto.core.cache.memcached
+
+    *(Optional)* Instance location URI can be customized::
+
+        kinto.cache_hosts = 127.0.0.1:11211 127.0.0.1:11212
+
+    :noindex:
+
+    """
+
+    def __init__(self, client, *args, **kwargs):
+        super(Cache, self).__init__(*args, **kwargs)
+        self._client = client
+
+    def initialize_schema(self, dry_run=False):
+        # Nothing to do.
+        pass
+
+    @wrap_memcached_error
+    def flush(self):
+        self._client.flush_all()
+
+    @wrap_memcached_error
+    def _get(self, key):
+        value = self._client.get(self.prefix + key)
+        if not value:
+            return None, 0
+        data = json.loads(value)
+        return data["value"], data["ttl"]
+
+    def ttl(self, key):
+        _, ttl = self._get(key)
+        val = ttl - time()
+        return floor(val)
+
+    def get(self, key):
+        value, _ = self._get(key)
+        return value
+
+    @wrap_memcached_error
+    def expire(self, key, ttl):
+        if ttl == 0:
+            self.delete(key)
+        else:
+            # We can't use touch here because we need to update the TTL value in the record.
+            value = self.get(key)
+            self.set(key, value, ttl)
+
+    @wrap_memcached_error
+    def set(self, key, value, ttl):
+        if isinstance(value, bytes):
+            raise TypeError("a string-like object is required, not 'bytes'")
+        value = json.dumps({"value": value, "ttl": ceil(time() + ttl)})
+        self._client.set(self.prefix + key, value, int(ttl))
+
+    @wrap_memcached_error
+    def delete(self, key):
+        value = self.get(key)
+        self._client.delete(self.prefix + key)
+        return value
+
+
+def load_from_config(config):
+    settings = config.get_settings()
+    client = create_from_config(config, prefix='cache_')
+    return Cache(client, cache_prefix=settings['cache_prefix'])
diff --git a/kinto/core/utils.py b/kinto/core/utils.py
--- a/kinto/core/utils.py
+++ b/kinto/core/utils.py
@@ -17,6 +17,11 @@
 except ImportError:  # pragma: no cover
     sqlalchemy = None
 
+try:
+    import memcache
+except ImportError:  # pragma: no cover
+    memcache = None
+
 from pyramid import httpexceptions
 from pyramid.interfaces import IRoutesMapper
 from pyramid.request import Request, apply_request_extensions
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -46,6 +46,10 @@ def read_file(filename):
     'kinto_redis'
 ]
 
+MEMCACHED_REQUIRES = [
+    'python-memcached'
+]
+
 SETUP_REQUIRES = [
     'pytest-runner'
 ]
@@ -104,6 +108,7 @@ def read_file(filename):
       install_requires=REQUIREMENTS,
       extras_require={
           'redis': REDIS_REQUIRES,
+          'memcached': MEMCACHED_REQUIRES,
           'postgresql': POSTGRESQL_REQUIRES,
           'monitoring': MONITORING_REQUIRES,
       },


Test patch:
diff --git a/kinto/core/testing.py b/kinto/core/testing.py
--- a/kinto/core/testing.py
+++ b/kinto/core/testing.py
@@ -11,10 +11,11 @@
 from kinto.core import DEFAULT_SETTINGS
 from kinto.core import statsd
 from kinto.core.storage import generators
-from kinto.core.utils import sqlalchemy, follow_subrequest, encode64
+from kinto.core.utils import sqlalchemy, memcache, follow_subrequest, encode64
 
 skip_if_travis = unittest.skipIf('TRAVIS' in os.environ, 'travis')
 skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, 'postgresql is not installed.')
+skip_if_no_memcached = unittest.skipIf(memcache is None, 'memcached is not installed.')
 skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, 'statsd is not installed.')
 
 
diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py
--- a/tests/core/test_cache.py
+++ b/tests/core/test_cache.py
@@ -2,11 +2,11 @@
 import unittest
 import time
 
-from kinto.core.utils import sqlalchemy
-from kinto.core.cache import (CacheBase, memory as memory_backend,
+from kinto.core.utils import sqlalchemy, memcache
+from kinto.core.cache import (CacheBase, memory as memory_backend, memcached as memcached_backend,
                               postgresql as postgresql_backend)
 from kinto.core.cache.testing import CacheTest
-from kinto.core.testing import skip_if_no_postgresql
+from kinto.core.testing import skip_if_no_postgresql, skip_if_no_memcached
 
 
 class CacheBaseTest(unittest.TestCase):
@@ -84,6 +84,35 @@ def test_size_quota_can_be_set_to_zero(self):
         assert self.cache.get('foobar') == 'tata'
 
 
+@skip_if_no_memcached
+class MemcachedCacheTest(CacheTest, unittest.TestCase):
+    backend = memcached_backend
+    settings = {
+        'cache_prefix': '',
+        'cache_hosts': '127.0.0.1:11211',
+    }
+
+    def setUp(self):
+        super().setUp()
+        self.client_error_patcher = mock.patch.object(
+            self.cache._client.servers[0],
+            'connect',
+            side_effect=memcache.Client.MemcachedKeyError)
+
+    def test_set_with_ttl_expires_the_value(self):
+        self.cache.set('foobar', 'toto', 1)
+        time.sleep(1.1)
+        retrieved = self.cache.get('foobar')
+        self.assertIsNone(retrieved)
+
+    def test_expire_expires_the_value(self):
+        self.cache.set('foobar', 'toto', 42)
+        self.cache.expire('foobar', 1)
+        time.sleep(1.1)
+        retrieved = self.cache.get('foobar')
+        self.assertIsNone(retrieved)
+
+
 @skip_if_no_postgresql
 class PostgreSQLCacheTest(CacheTest, unittest.TestCase):
     backend = postgresql_backend
diff --git a/tests/test_configuration/test.ini b/tests/test_configuration/test.ini
--- a/tests/test_configuration/test.ini
+++ b/tests/test_configuration/test.ini
@@ -62,6 +62,9 @@ kinto.cache_url = cache_url
 # kinto.cache_pool_timeout = 30
 # kinto.cache_max_backlog = -1
 
+# kinto.cache_backend = kinto.core.cache.memcached
+# kinto.cache_hosts = 127.0.0.1:11211
+
 # Permissions.
 # https://kinto.readthedocs.io/en/latest/configuration/settings.html#permissions
 #