Repository: Kinto/kinto

Issue:
Add limit for cache size (especially memory)
original: https://github.com/mozilla-services/cliquet/issues/406

That's an improvement so we control the cache size. Otherwise it's a free wheel that eventually crashes the server. 

Which makes me think: if I do a load test on stage today, is the kinto process growing indefinitely in memory ?



Discussion:
> into process growing indefinitely in memory ?

We use postgresql on stage AFAIK so no problem about crashing the web head I believe

what about other back-ends ? do we care ? or should we state explicitly in our docs they are not suited for production use ?

I think you are right we need to set cache limit somewhere. That's mandatory for cache right?
Then I have no idea of how we should handle a cache oversize? Should we raise ? Should we delete some old data? How?

Limiting cache size applies to every backend I think. Writing to cache could prune the oldest data until the total size is bigger than a limit. 

> Should we raise ? 

No, caching should not impact the result of the web service calls. just speed it. So if the cache is full, it's up to the application to do whatever it wants to optimize it. 

Sorting by date is one option, another one could be by hits e.g. getting rid of the least called cached entries.

This requires:
- Generalize storage migration mecanism to cache backend
- Add a hits attribute to track calls to cached entries
- Add a created attribute to track insertion/refresh
- Compute size of value and denormalize total
- Before inserting: if total > max_size then sorte entries by hits+created and delete keys until total < max_size


Resolving patch:
diff --git a/kinto/core/__init__.py b/kinto/core/__init__.py
--- a/kinto/core/__init__.py
+++ b/kinto/core/__init__.py
@@ -27,6 +27,7 @@
     'cache_url': '',
     'cache_pool_size': 25,
     'cache_prefix': '',
+    'cache_max_size_bytes': 524288,
     'cors_origins': '*',
     'cors_max_age_seconds': 3600,
     'eos': None,
diff --git a/kinto/core/cache/__init__.py b/kinto/core/cache/__init__.py
--- a/kinto/core/cache/__init__.py
+++ b/kinto/core/cache/__init__.py
@@ -12,7 +12,7 @@ class CacheBase(object):
 
     def __init__(self, *args, **kwargs):
         self.prefix = kwargs['cache_prefix']
-        pass
+        self.max_size_bytes = kwargs.get('cache_max_size_bytes')
 
     def initialize_schema(self, dry_run=False):
         """Create every necessary objects (like tables or indices) in the
diff --git a/kinto/core/cache/memory.py b/kinto/core/cache/memory.py
--- a/kinto/core/cache/memory.py
+++ b/kinto/core/cache/memory.py
@@ -21,8 +21,25 @@ def initialize_schema(self, dry_run=False):
         pass
 
     def flush(self):
+        self._created_at = {}
         self._ttl = {}
         self._store = {}
+        self._quota = 0
+
+    def _clean_expired(self):
+        current = msec_time()
+        expired = [k for k, v in self._ttl.items() if current >= v]
+        for expired_item_key in expired:
+            self.delete(expired_item_key[len(self.prefix):])
+
+    def _clean_oversized(self):
+        if self._quota < self.max_size_bytes:
+            return
+
+        for key, value in sorted(self._created_at.items(), key=lambda k: k[1]):
+            if self._quota < (self.max_size_bytes * 0.8):
+                break
+            self.delete(key[len(self.prefix):])
 
     @synchronized
     def ttl(self, key):
@@ -37,25 +54,36 @@ def expire(self, key, ttl):
 
     @synchronized
     def set(self, key, value, ttl=None):
+        self._clean_expired()
+        self._clean_oversized()
         if ttl is not None:
             self.expire(key, ttl)
-        self._store[self.prefix + key] = value
+        item_key = self.prefix + key
+        self._store[item_key] = value
+        self._created_at[item_key] = msec_time()
+        self._quota += size_of(item_key, value)
 
     @synchronized
     def get(self, key):
-        current = msec_time()
-        expired = [k for k, v in self._ttl.items() if current >= v]
-        for expired_item_key in expired:
-            self.delete(expired_item_key[len(self.prefix):])
+        self._clean_expired()
         return self._store.get(self.prefix + key)
 
     @synchronized
     def delete(self, key):
         key = self.prefix + key
         self._ttl.pop(key, None)
-        self._store.pop(key, None)
+        self._created_at.pop(key, None)
+        value = self._store.pop(key, None)
+        self._quota -= size_of(key, value)
 
 
 def load_from_config(config):
     settings = config.get_settings()
-    return Cache(cache_prefix=settings['cache_prefix'])
+    return Cache(cache_prefix=settings['cache_prefix'],
+                 cache_max_size_bytes=settings['cache_max_size_bytes'])
+
+
+def size_of(key, value):
+    # Key used for ttl, created_at and store.
+    # Int size is 24 bytes one for ttl and one for created_at values
+    return len(key) * 3 + len(str(value)) + 24 * 2


Test patch:
diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py
--- a/tests/core/test_cache.py
+++ b/tests/core/test_cache.py
@@ -1,5 +1,6 @@
 import mock
 import unittest
+import time
 
 from kinto.core.utils import sqlalchemy
 from kinto.core.cache import (CacheBase, memory as memory_backend,
@@ -30,7 +31,8 @@ def test_mandatory_overrides(self):
 class MemoryCacheTest(CacheTest, unittest.TestCase):
     backend = memory_backend
     settings = {
-        'cache_prefix': ''
+        'cache_prefix': '',
+        'cache_max_size_bytes': 7000
     }
 
     def get_backend_prefix(self, prefix):
@@ -51,6 +53,30 @@ def test_ping_returns_false_if_unavailable(self):
     def test_ping_logs_error_if_unavailable(self):
         pass
 
+    def test_clean_expired_expires_items(self):
+        self.cache.set('foobar', 'toto', 0.01)
+        assert 'foobar' in self.cache._store
+        assert 'foobar' in self.cache._ttl
+        assert 'foobar' in self.cache._created_at
+        time.sleep(0.02)
+        retrieved = self.cache._clean_expired()
+        assert 'foobar' not in self.cache._store
+        assert 'foobar' not in self.cache._ttl
+        assert 'foobar' not in self.cache._created_at
+        self.assertIsNone(retrieved)
+
+    def test_add_over_quota_clean_oversized_items(self):
+        for x in range(100):
+            # Each entry is 70 bytes
+            self.cache.set('foo' + str(x).zfill(3), 'toto')
+            time.sleep(0.001)
+        assert self.cache.get('foo000') == 'toto'
+        # This should delete the 2 first entries
+        self.cache.set('foobar', 'tata')
+        assert self.cache._quota == 7000 - 70 * 20
+        assert self.cache.get('foo000') is None
+        assert self.cache.get('foobar') == 'tata'
+
 
 @skip_if_no_postgresql
 class PostgreSQLCacheTest(CacheTest, unittest.TestCase):
diff --git a/tests/test_configuration/test.ini b/tests/test_configuration/test.ini
--- a/tests/test_configuration/test.ini
+++ b/tests/test_configuration/test.ini
@@ -20,6 +20,7 @@ kinto.storage_backend = storage_backend
 kinto.storage_url = storage_url
 kinto.cache_backend = cache_backend
 kinto.cache_url = cache_url
+# kinto.cache_max_size_bytes = 524288
 kinto.permission_backend = permission_backend
 kinto.permission_url = permission_url