Repository: Kinto/kinto

Issue:
Heartbeat max timeout
Let's make sure the heartbeat times out before 30 seconds. Otherwise uWSGI hara kiris the web head and we don't have a chance to send back the result of the heartbeat where it tells us which backend fails.

I suggest 15 seconds max by defaut + an option.



Discussion:
I though it was already the case do you know for which backend it is not the case?

The signer blocks for 30s. This should be enforced by the whole heartbeat view, not delegated by each backend. Because 30s + 30s + 30s... :)


Resolving patch:
diff --git a/kinto/core/__init__.py b/kinto/core/__init__.py
--- a/kinto/core/__init__.py
+++ b/kinto/core/__init__.py
@@ -54,6 +54,7 @@
         'kinto.core.events.setup_transaction_hook',
     ),
     'event_listeners': '',
+    'heartbeat_timeout_seconds': 10,
     'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',
     'newrelic_config': None,
     'newrelic_env': 'dev',
diff --git a/kinto/core/views/heartbeat.py b/kinto/core/views/heartbeat.py
--- a/kinto/core/views/heartbeat.py
+++ b/kinto/core/views/heartbeat.py
@@ -1,7 +1,10 @@
+from concurrent.futures import ThreadPoolExecutor, wait
 from pyramid.security import NO_PERMISSION_REQUIRED
 
+from kinto import logger
 from kinto.core import Service
 
+
 heartbeat = Service(name="heartbeat", path='/__heartbeat__',
                     description="Server health")
 
@@ -11,10 +14,38 @@ def get_heartbeat(request):
     """Return information about server health."""
     status = {}
 
+    def heartbeat_check(name, func):
+        status[name] = False
+        status[name] = func(request)
+
+    # Start executing heartbeats concurrently.
     heartbeats = request.registry.heartbeats
-    for name, callable in heartbeats.items():
-        status[name] = callable(request)
+    pool = ThreadPoolExecutor(max_workers=max(1, len(heartbeats.keys())))
+    futures = []
+    for name, func in heartbeats.items():
+        future = pool.submit(heartbeat_check, name, func)
+        future.__heartbeat_name = name  # For logging purposes.
+        futures.append(future)
+
+    # Wait for the results, with timeout.
+    seconds = float(request.registry.settings['heartbeat_timeout_seconds'])
+    done, not_done = wait(futures, timeout=seconds)
+
+    # A heartbeat is supposed to return True or False, and never raise.
+    # Just in case, go though results to spot any potential exception.
+    for future in done:
+        exc = future.exception()
+        if exc is not None:
+            logger.error("%r heartbeat failed." % future.__heartbeat_name)
+            logger.error(exc)
+
+    # Log timed-out heartbeats.
+    for future in not_done:
+        name = future.__heartbeat_name
+        error_msg = "%r heartbeat has exceeded timeout of %s seconds."
+        logger.error(error_msg % (name, seconds))
 
+    # If any has failed, return a 503 error response.
     has_error = not all([v or v is None for v in status.values()])
     if has_error:
         request.response.status = 503
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -100,7 +100,7 @@ def read_file(filename):
       extras_require={
           'postgresql': POSTGRESQL_REQUIRES,
           'monitoring': MONITORING_REQUIRES,
-          ":python_version=='2.7'": ["functools32"],
+          ":python_version=='2.7'": ["functools32", "futures"],
       },
       test_suite="kinto.tests",
       dependency_links=DEPENDENCY_LINKS,


Test patch:
diff --git a/kinto/tests/core/test_views_heartbeat.py b/kinto/tests/core/test_views_heartbeat.py
--- a/kinto/tests/core/test_views_heartbeat.py
+++ b/kinto/tests/core/test_views_heartbeat.py
@@ -1,24 +1,15 @@
 import mock
-import requests
 
 from .support import BaseWebTest, unittest
 
 
-httpOK = requests.models.Response()
-httpOK.status_code = 200
-
-
 class SuccessTest(BaseWebTest, unittest.TestCase):
 
-    @mock.patch('requests.get')
-    def test_returns_storage_true_if_ok(self, get_mocked):
-        get_mocked.return_value = httpOK
+    def test_returns_storage_true_if_ok(self):
         response = self.app.get('/__heartbeat__')
         self.assertEqual(response.json['storage'], True)
 
-    @mock.patch('requests.get')
-    def test_returns_cache_true_if_ok(self, get_mocked):
-        get_mocked.return_value = httpOK
+    def test_returns_cache_true_if_ok(self):
         response = self.app.get('/__heartbeat__')
         self.assertEqual(response.json['cache'], True)
 
@@ -30,19 +21,34 @@ def test_successful_if_one_heartbeat_is_none(self):
 
 class FailureTest(BaseWebTest, unittest.TestCase):
 
-    @mock.patch('requests.get')
-    def test_returns_storage_false_if_ko(self, get_mocked):
+    def test_returns_storage_false_if_ko(self):
         self.app.app.registry.heartbeats['storage'] = lambda r: False
-        get_mocked.return_value = httpOK
         response = self.app.get('/__heartbeat__', status=503)
         self.assertEqual(response.json['storage'], False)
+        self.assertEqual(response.json['cache'], True)
 
-    @mock.patch('requests.get')
-    def test_returns_cache_false_if_ko(self, get_mocked):
+    def test_returns_cache_false_if_ko(self):
         self.app.app.registry.heartbeats['cache'] = lambda r: False
-        get_mocked.return_value = httpOK
         response = self.app.get('/__heartbeat__', status=503)
         self.assertEqual(response.json['cache'], False)
+        self.assertEqual(response.json['storage'], True)
+
+    def test_returns_false_if_heartbeat_times_out(self):
+        def sleepy(request):
+            import time
+            time.sleep(1)
+        self.app.app.registry.heartbeats['cache'] = sleepy
+        with mock.patch.dict(self.app.app.registry.settings,
+                             [('heartbeat_timeout_seconds', 0.1)]):
+            response = self.app.get('/__heartbeat__', status=503)
+        self.assertEqual(response.json['cache'], False)
+        self.assertEqual(response.json['storage'], True)
+
+    def test_returns_false_if_heartbeat_fails(self):
+        self.app.app.registry.heartbeats['cache'] = lambda r: 1 / 0
+        response = self.app.get('/__heartbeat__', status=503)
+        self.assertEqual(response.json['cache'], False)
+        self.assertEqual(response.json['storage'], True)
 
 
 class LoadBalancerHeartbeat(BaseWebTest, unittest.TestCase):