Repository: deis/deis

Issue:
Application healthcheck endpoint for checks during deploy and publishing
During an application release/deploy, the container just checks to make sure that the fleet unit reports that it is running: https://github.com/deis/deis/blob/c2cffb969a1edaaa456033a05b7a6958eea02bad/controller/scheduler/fleet.py#L193

Just because a unit is running doesn't mean that it's ready/healthy.

Additionally, there is actually a bug in the way that publisher checks for applications to be listening before it publishes them - @gabrtv discovered that the Docker proxy responsible for routing requests from the host into containers will actually accept on the external port immediately (even if no daemon in the container is bound to that port). 

The proposal is to:
- Add a new `healthcheck` endpoint and CLI command to configure a healthcheck endpoint for the application
- Modify the controller so that during a deploy, controller will wait for each deployed container to report healthy if a healthcheck endpoint is configured for the application
- Modify the publisher to check for an appropriate (HTTP 200?) response from the healthcheck endpoint before publishing an app container if a healthcheck endpoint is configured for the application

This solution addresses several issues and feature requests surrounding application deploys and health checks in the router, including #3478, #3723, and #3674.



Discussion:

Resolving patch:
diff --git a/controller/api/models.py b/controller/api/models.py
--- a/controller/api/models.py
+++ b/controller/api/models.py
@@ -299,6 +299,45 @@ def _start_containers(self, to_add):
         if set([c.state for c in to_add]) != set(['up']):
             err = 'warning, some containers failed to start'
             log_event(self, err, logging.WARNING)
+        # if the user specified a health check, try checking to see if it's running
+        try:
+            config = self.config_set.latest()
+            if 'HEALTHCHECK_URL' in config.values.keys():
+                # check the app to see if it's healthy
+                try:
+                    self._do_healthcheck(config)
+                except Exception as e:
+                    log_event(self, str(e), logging.ERROR)
+                    self._destroy_containers(to_add)
+                    raise
+        except Config.DoesNotExist:
+            pass
+
+    def _do_healthcheck(self, config):
+        timeout = time.time() + 60  # 1 minute from now
+        while True:
+            if time.time() > timeout:
+                raise RuntimeError(
+                    'app failed to respond to health check within 60 seconds of launch')
+            try:
+                response = requests.get(
+                    'http://{}{}'.format(
+                        self.url,
+                        config.values['HEALTHCHECK_URL']),
+                    timeout=5)
+                expected_status_code = config.values['HEALTHCHECK_STATUS_CODE'] if \
+                    'HEALTHCHECK_STATUS_CODE' in config.values.keys() else \
+                    settings.DEIS_HEALTHCHECK_STATUS_CODE
+                if str(response.status_code) != str(expected_status_code):
+                    err = "aborting, app failed health check (got '{}', expected: '{}')".format(
+                        response.status_code,
+                        expected_status_code)
+                    raise RuntimeError(err)
+                break
+            except requests.exceptions.ConnectionError:
+                continue
+            except requests.exceptions.Timeout:
+                continue
 
     def _restart_containers(self, to_restart):
         """Restarts containers via the scheduler"""
@@ -1004,6 +1043,34 @@ def _etcd_purge_app(**kwargs):
         pass
 
 
+def _etcd_publish_config(**kwargs):
+    config = kwargs['instance']
+    # we purge all existing config when adding the newest instance. This is because
+    # deis config:unset would remove an existing value, but not delete the
+    # old config object
+    try:
+        _etcd_client.delete('/deis/services/{}/config'.format(config.app),
+                            prevExist=True, dir=True, recursive=True)
+    except KeyError:
+        pass
+    if kwargs['created']:
+        for k, v in config.values.iteritems():
+            _etcd_client.write(
+                '/deis/services/{}/config/{}'.format(
+                    config.app,
+                    unicode(k).encode('utf-8').lower()),
+                unicode(v).encode('utf-8'))
+
+
+def _etcd_purge_config(**kwargs):
+    config = kwargs['instance']
+    try:
+        _etcd_client.delete('/deis/services/{}/config'.format(config.app),
+                            prevExist=True, dir=True, recursive=True)
+    except KeyError:
+        pass
+
+
 def _etcd_publish_cert(**kwargs):
     cert = kwargs['instance']
     if kwargs['created']:
@@ -1069,3 +1136,5 @@ def create_auth_token(sender, instance=None, created=False, **kwargs):
     post_delete.connect(_etcd_purge_app, sender=App, dispatch_uid='api.models')
     post_save.connect(_etcd_publish_cert, sender=Certificate, dispatch_uid='api.models')
     post_delete.connect(_etcd_purge_cert, sender=Certificate, dispatch_uid='api.models')
+    post_save.connect(_etcd_publish_config, sender=Config, dispatch_uid='api.models')
+    post_delete.connect(_etcd_purge_config, sender=Config, dispatch_uid='api.models')
diff --git a/controller/deis/settings.py b/controller/deis/settings.py
--- a/controller/deis/settings.py
+++ b/controller/deis/settings.py
@@ -290,6 +290,9 @@
 # names which apps cannot reserve for routing
 DEIS_RESERVED_NAMES = ['deis']
 
+# the default expected status code for healthchecks
+DEIS_HEALTHCHECK_STATUS_CODE = '200'
+
 # default scheduler settings
 SCHEDULER_MODULE = 'scheduler.mock'
 SCHEDULER_TARGET = ''  # path to scheduler endpoint (e.g. /var/run/fleet.sock)


Test patch:
diff --git a/controller/api/tests/test_config.py b/controller/api/tests/test_config.py
--- a/controller/api/tests/test_config.py
+++ b/controller/api/tests/test_config.py
@@ -15,16 +15,38 @@
 from django.test import TransactionTestCase
 from rest_framework.authtoken.models import Token
 
-from api.models import Config
+from api.models import App, Config
 
 
-def mock_import_repository_task(*args, **kwargs):
+def mock_status_ok(*args, **kwargs):
     resp = requests.Response()
     resp.status_code = 200
     resp._content_consumed = True
     return resp
 
 
+def mock_status_not_found(*args, **kwargs):
+    resp = requests.Response()
+    resp.status_code = 404
+    resp._content_consumed = True
+    return resp
+
+
+def mock_request_timed_out(*args, **kwargs):
+    raise requests.exceptions.Timeout()
+
+
+def mock_request_connection_error(*args, **kwargs):
+    raise requests.exceptions.ConnectionError()
+
+
+def mock_time(*args, **kwargs):
+    if not hasattr(mock_time, "counter"):
+        mock_time.counter = 0  # it doesn't exist yet, so initialize it
+    mock_time.counter += 1
+    return mock_time.counter
+
+
 class ConfigTest(TransactionTestCase):
 
     """Tests setting and updating config values"""
@@ -35,7 +57,7 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = Token.objects.get(user=self.user).key
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_config(self):
         """
         Test that config is auto-created for a new app and that
@@ -107,7 +129,7 @@ def test_config(self):
         self.assertEqual(response.status_code, 405)
         return config5
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_response_data(self):
         """Test that the serialized response contains only relevant data."""
         body = {'id': 'test'}
@@ -132,7 +154,7 @@ def test_response_data(self):
         }
         self.assertDictContainsSubset(expected, response.data)
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_config_set_same_key(self):
         """
         Test that config sets on the same key function properly
@@ -156,7 +178,7 @@ def test_config_set_same_key(self):
         self.assertIn('PORT', response.data['values'])
         self.assertEqual(response.data['values']['PORT'], '5001')
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_config_set_unicode(self):
         """
         Test that config sets with unicode values are accepted.
@@ -187,14 +209,14 @@ def test_config_set_unicode(self):
         self.assertIn('INTEGER', response.data['values'])
         self.assertEqual(response.data['values']['INTEGER'], 1)
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_config_str(self):
         """Test the text representation of a node."""
         config5 = self.test_config()
         config = Config.objects.get(uuid=config5['uuid'])
         self.assertEqual(str(config), "{}-{}".format(config5['app'], config5['uuid'][:7]))
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_admin_can_create_config_on_other_apps(self):
         """If a non-admin creates an app, an administrator should be able to set config
         values for that app.
@@ -214,7 +236,7 @@ def test_admin_can_create_config_on_other_apps(self):
         self.assertIn('PORT', response.data['values'])
         return response
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_limit_memory(self):
         """
         Test that limit is auto-created for a new app and that
@@ -302,7 +324,7 @@ def test_limit_memory(self):
         self.assertEqual(response.status_code, 405)
         return limit4
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_limit_cpu(self):
         """
         Test that CPU limits can be set
@@ -373,7 +395,7 @@ def test_limit_cpu(self):
         self.assertEqual(response.status_code, 405)
         return limit4
 
-    @mock.patch('requests.post', mock_import_repository_task)
+    @mock.patch('requests.post', mock_status_ok)
     def test_tags(self):
         """
         Test that tags can be set on an application
@@ -477,3 +499,73 @@ def test_unauthorized_user_cannot_modify_config(self):
         response = self.client.post(url, json.dumps(body), content_type='application/json',
                                     HTTP_AUTHORIZATION='token {}'.format(unauthorized_token))
         self.assertEqual(response.status_code, 403)
+
+    def _test_app_healthcheck(self):
+        url = '/v1/apps'
+        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
+        self.assertEqual(response.status_code, 201)
+        app_id = response.data['id']
+        # post a new build, expecting it to pass as usual
+        url = "/v1/apps/{app_id}/builds".format(**locals())
+        body = {'image': 'autotest/example'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json',
+                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
+        self.assertEqual(response.status_code, 201)
+        # set an initial healthcheck url.
+        url = "/v1/apps/{app_id}/config".format(**locals())
+        body = {'values': json.dumps({'HEALTHCHECK_URL': '/'})}
+        return self.client.post(url, json.dumps(body), content_type='application/json',
+                                HTTP_AUTHORIZATION='token {}'.format(self.token))
+
+    @mock.patch('requests.get', mock_status_not_found)
+    def test_app_healthcheck_good(self):
+        """
+        If a user deploys an app with a config value set for HEALTHCHECK_URL, the controller
+        should check that the application is up. If it's down, the app should be rolled back.
+        """
+        response = self._test_app_healthcheck()
+        self.assertEqual(response.status_code, 503)
+        self.assertEqual(
+            response.data,
+            {
+                'detail':
+                "aborting, app failed health check (got '404', expected: '200')"
+            })
+        # add in the expected app healthcheck status, which will result in a successful deployment
+        app_id = App.objects.all()[0]
+        url = "/v1/apps/{app_id}/config".format(**locals())
+        body = {'values': json.dumps({'HEALTHCHECK_STATUS_CODE': '404'})}
+        response = self.client.post(url, json.dumps(body), content_type='application/json',
+                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
+        self.assertEqual(response.status_code, 201)
+
+    @mock.patch('requests.get', mock_request_timed_out)
+    @mock.patch('time.time', mock_time)
+    def test_app_healthcheck_timeout(self):
+        """
+        If a user deploys an app with a config value set for HEALTHCHECK_URL but the app
+        times out, the controller should continue checking until either the app
+        responds or the app fails to respond within the timeout.
+        """
+        response = self._test_app_healthcheck()
+        self.assertEqual(response.status_code, 503)
+        self.assertEqual(
+            response.data,
+            {'detail': 'app failed to respond to health check within 60 seconds of launch'})
+
+    @mock.patch('requests.get', mock_request_connection_error)
+    @mock.patch('time.time', mock_time)
+    def test_app_healthcheck_connection_error(self):
+        """
+        If a user deploys an app with a config value set for HEALTHCHECK_URL but the app
+        returns a connection error, the controller should continue checking until either the app
+        responds or the app fails to respond within the timeout.
+
+        NOTE (bacongobbler): the Docker userland proxy listens for connections and returns a
+        ConnectionError, hence the unit test.
+        """
+        response = self._test_app_healthcheck()
+        self.assertEqual(response.status_code, 503)
+        self.assertEqual(
+            response.data,
+            {'detail': 'app failed to respond to health check within 60 seconds of launch'})
diff --git a/publisher/server/publisher_test.go b/publisher/server/publisher_test.go
--- a/publisher/server/publisher_test.go
+++ b/publisher/server/publisher_test.go
@@ -1,7 +1,10 @@
 package server
 
 import (
+	"fmt"
 	"net"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 )
 
@@ -45,3 +48,18 @@ func TestIsPortOpen(t *testing.T) {
 		t.Errorf("Port should be closed")
 	}
 }
+
+func TestHealthCheckOK(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello, client")
+	}))
+	defer ts.Close()
+
+	s := &Server{}
+	if !s.HealthCheckOK(ts.URL, 200) {
+		t.Errorf("Health check should be OK")
+	}
+	if s.HealthCheckOK("127.0.0.1:-1", 200) {
+		t.Errorf("Health check should be NOT OK")
+	}
+}
diff --git a/tests/config_test.go b/tests/config_test.go
--- a/tests/config_test.go
+++ b/tests/config_test.go
@@ -11,12 +11,13 @@ import (
 )
 
 var (
-	configListCmd         = "config:list --app={{.AppName}}"
-	configSetCmd          = "config:set FOO=讲台 --app={{.AppName}}"
-	configSet2Cmd         = "config:set FOO=10 --app={{.AppName}}"
-	configSet3Cmd         = "config:set POWERED_BY=\"the Deis team\" --app={{.AppName}}"
-	configSetBuildpackCmd = "config:set BUILDPACK_URL=$BUILDPACK_URL --app={{.AppName}}"
-	configUnsetCmd        = "config:unset FOO --app={{.AppName}}"
+	configListCmd           = "config:list --app={{.AppName}}"
+	configSetCmd            = "config:set FOO=讲台 --app={{.AppName}}"
+	configSet2Cmd           = "config:set FOO=10 --app={{.AppName}}"
+	configSet3Cmd           = "config:set POWERED_BY=\"the Deis team\" --app={{.AppName}}"
+	configSetBuildpackCmd   = "config:set BUILDPACK_URL=$BUILDPACK_URL --app={{.AppName}}"
+	configSetHealthcheckCmd = "config:set HEALTHCHECK_URL=/ --app={{.AppName}}"
+	configUnsetCmd          = "config:unset FOO --app={{.AppName}}"
 )
 
 var buildpacks = map[string]string{
@@ -90,6 +91,8 @@ func configSetTest(t *testing.T, params *utils.DeisTestConfig) {
 	utils.CheckList(t, appsInfoCmd, params, "(v5)", false)
 	utils.Execute(t, configSet2Cmd, params, false, "10")
 	utils.CheckList(t, appsInfoCmd, params, "(v6)", false)
+	utils.Execute(t, configSetHealthcheckCmd, params, false, "/")
+	utils.CheckList(t, appsInfoCmd, params, "(v7)", false)
 }
 
 func configPushTest(t *testing.T, params *utils.DeisTestConfig) {
@@ -101,7 +104,7 @@ func configPushTest(t *testing.T, params *utils.DeisTestConfig) {
 		t.Fatal(err)
 	}
 	utils.Execute(t, "config:push --app {{.AppName}}", params, false, "Deis")
-	utils.CheckList(t, appsInfoCmd, params, "(v7)", false)
+	utils.CheckList(t, appsInfoCmd, params, "(v8)", false)
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatal(err)
 	}
@@ -109,6 +112,6 @@ func configPushTest(t *testing.T, params *utils.DeisTestConfig) {
 
 func configUnsetTest(t *testing.T, params *utils.DeisTestConfig) {
 	utils.Execute(t, configUnsetCmd, params, false, "")
-	utils.CheckList(t, appsInfoCmd, params, "(v8)", false)
+	utils.CheckList(t, appsInfoCmd, params, "(v9)", false)
 	utils.CheckList(t, "run env --app={{.AppName}}", params, "FOO", true)
 }