Repository: deis/deis

Issue:
improve fleet error handling
The scheduler currently does not handle any errors that occur when shelling out fleetctl commands from the controller. They should be handled in a proper fashion a la Exception/Error handling. See https://github.com/opdemand/deis/blob/scheduler/controller/scheduler/coreos.py#L99-L100 for more info.

Viewing containers in Django admin raises 500 error
Bring up a vagrant installation and register, then log in to the web UI as the initial admin user. This URL in the Django admin will error: http://local.deisapp.com:8000/admin/api/container/

The logs in the `make run` terminal say:

``` console
...
Apr 16 18:39:23 deis sh[16287]: fields.append(parent._meta.get_field_by_name(piece)[0])
Apr 16 18:39:23 deis sh[16287]: File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 377, in get_field_by_name
Apr 16 18:39:23 deis sh[16287]: % (self.object_name, name))
Apr 16 18:39:23 deis sh[16287]: FieldDoesNotExist: Container has no field named u'cluster'
```

This is probably a matter of updating the declarative fields in controller/api/admin.py.



Discussion:
@gabrtv, how do you want to do this? Calls with `subprocess.Popen` dump stdout and stderr to `subprocess.PIPE`, and any call with `subprocess.check_call()` that returns non-zero should already raise a CalledProcessError. See https://docs.python.org/2/library/subprocess.html#subprocess.CalledProcessError

After talking with @gabrtv about this, we need some way to test that the scheduler and the containers themselves transition state as expected to (i.e. if a container crashes while deploying, it should go into an error state).

A `Container` does not have a cluster attribute, but an `App` does. I'll ninja-fix this.


Resolving patch:
diff --git a/controller/api/admin.py b/controller/api/admin.py
--- a/controller/api/admin.py
+++ b/controller/api/admin.py
@@ -63,8 +63,8 @@ class ContainerAdmin(admin.ModelAdmin):
     in the Django admin.
     """
     date_hierarchy = 'created'
-    list_display = ('short_name', 'owner', 'cluster', 'app', 'state')
-    list_filter = ('owner', 'cluster', 'app', 'state')
+    list_display = ('short_name', 'owner', 'app', 'state')
+    list_filter = ('owner', 'app', 'state')
 admin.site.register(Container, ContainerAdmin)
 
 
diff --git a/controller/api/models.py b/controller/api/models.py
--- a/controller/api/models.py
+++ b/controller/api/models.py
@@ -63,11 +63,12 @@ class Cluster(UuidAuditedModel):
     """
 
     CLUSTER_TYPES = (('mock', 'Mock Cluster'),
-                     ('coreos', 'CoreOS Cluster'))
+                     ('coreos', 'CoreOS Cluster'),
+                     ('faulty', 'Faulty Cluster'))
 
     owner = models.ForeignKey(settings.AUTH_USER_MODEL)
     id = models.CharField(max_length=128, unique=True)
-    type = models.CharField(max_length=16, choices=CLUSTER_TYPES)
+    type = models.CharField(max_length=16, choices=CLUSTER_TYPES, default='coreos')
 
     domain = models.CharField(max_length=128)
     hosts = models.CharField(max_length=256)
diff --git a/controller/scheduler/faulty.py b/controller/scheduler/faulty.py
new file mode 100644
--- /dev/null
+++ b/controller/scheduler/faulty.py
@@ -0,0 +1,31 @@
+class FaultyClient(object):
+    """A faulty scheduler that will always fail"""
+
+    def __init__(self, cluster_name, hosts, auth, domain, options):
+        pass
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def create(self, name, image, command='', template=None, port=5000):
+        raise Exception()
+
+    def start(self, name):
+        raise Exception()
+
+    def stop(self, name):
+        raise Exception()
+
+    def destroy(self, name):
+        raise Exception()
+
+    def run(self, name, image, command):
+        raise Exception()
+
+    def attach(self, name):
+        raise Exception()
+
+SchedulerClient = FaultyClient


Test patch:
diff --git a/controller/api/tests/test_container.py b/controller/api/tests/test_container.py
--- a/controller/api/tests/test_container.py
+++ b/controller/api/tests/test_container.py
@@ -12,6 +12,8 @@
 from django.test import TransactionTestCase
 from django.test.utils import override_settings
 
+from django_fsm import TransitionNotAllowed
+
 from api.models import Container, App
 
 
@@ -30,8 +32,15 @@ def setUp(self):
         response = self.client.post('/api/clusters', json.dumps(body),
                                     content_type='application/json')
         self.assertEqual(response.status_code, 201)
+        # create a malicious scheduler as well
+        body['id'] = 'autotest2'
+        body['type'] = 'faulty'
+        response = self.client.post('/api/clusters', json.dumps(body),
+                                    content_type='application/json')
+        self.assertEqual(response.status_code, 201)
 
-    def test_container(self):
+    def test_container_state_good(self):
+        """Test that the finite state machine transitions with a good scheduler"""
         url = '/api/apps'
         body = {'cluster': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
@@ -44,13 +53,62 @@ def test_container(self):
                                      type='web',
                                      num=1)
         self.assertEqual(c.state, 'initialized')
+        # test an illegal transition
+        self.assertRaises(TransitionNotAllowed, lambda: c.start())
         c.create()
         self.assertEqual(c.state, 'created')
         c.start()
         self.assertEqual(c.state, 'up')
+        c.deploy(App.objects.get(id=app_id).release_set.latest())
+        self.assertEqual(c.state, 'up')
         c.destroy()
         self.assertEqual(c.state, 'destroyed')
 
+    def test_container_state_bad(self):
+        """Test that the finite state machine transitions with a faulty scheduler"""
+        url = '/api/apps'
+        body = {'cluster': 'autotest2'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        app_id = response.data['id']
+        # create a container
+        c = Container.objects.create(owner=User.objects.get(username='autotest'),
+                                     app=App.objects.get(id=app_id),
+                                     release=App.objects.get(id=app_id).release_set.latest(),
+                                     type='web',
+                                     num=1)
+        self.assertEqual(c.state, 'initialized')
+        self.assertRaises(Exception, lambda: c.create())
+        self.assertEqual(c.state, 'initialized')
+        # test an illegal transition
+        self.assertRaises(TransitionNotAllowed, lambda: c.start())
+        self.assertEqual(c.state, 'initialized')
+        self.assertRaises(
+            Exception,
+            lambda: c.deploy(
+                App.objects.get(id=app_id).release_set.latest()
+            )
+        )
+        self.assertEqual(c.state, 'down')
+        self.assertRaises(Exception, lambda: c.destroy())
+        self.assertEqual(c.state, 'down')
+        self.assertRaises(Exception, lambda: c.run('echo hello world'))
+        self.assertEqual(c.state, 'down')
+
+    def test_container_state_protected(self):
+        """Test that you cannot directly modify the state"""
+        url = '/api/apps'
+        body = {'cluster': 'autotest'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        app_id = response.data['id']
+        c = Container.objects.create(owner=User.objects.get(username='autotest'),
+                                     app=App.objects.get(id=app_id),
+                                     release=App.objects.get(id=app_id).release_set.latest(),
+                                     type='web',
+                                     num=1)
+        self.assertRaises(AttributeError, lambda: setattr(c, 'state', 'up'))
+
     def test_container_api(self):
         url = '/api/apps'
         body = {'cluster': 'autotest'}