Repository: jupyterhub/jupyterhub

Issue:
inconsistent db state when spawner fails to start
it's possible for some parts of the Hub to think a server is running when it's not, preventing stop/restart from working.

/cc @jhamrick



Discussion:
@minrk what does "some parts of the hub" mean here?  There's definitely a window from https://github.com/jupyter/jupyterhub/blob/master/jupyterhub/orm.py#L300 to https://github.com/jupyter/jupyterhub/blob/master/jupyterhub/orm.py#L333 where there's a server entry for the user but the server hasn't actually come up yet.  Is that the issue you're referring to, or is there something else you're seeing?

Apparently some APIs are using different information to report whether there's already a server running, so the admin page shows a 'stop' button, but clicking it results in "server isn't running".

This state can happen after a spawner fails to start, not just the spawn pending state, which isn't a problem.


Resolving patch:
diff --git a/jupyterhub/apihandlers/users.py b/jupyterhub/apihandlers/users.py
--- a/jupyterhub/apihandlers/users.py
+++ b/jupyterhub/apihandlers/users.py
@@ -14,12 +14,18 @@
 class BaseUserHandler(APIHandler):
     
     def user_model(self, user):
-        return {
+        model = {
             'name': user.name,
             'admin': user.admin,
-            'server': user.server.base_url if user.server and not (user.spawn_pending or user.stop_pending) else None,
+            'server': user.server.base_url if user.running else None,
+            'pending': None,
             'last_activity': user.last_activity.isoformat(),
         }
+        if user.spawn_pending:
+            model['pending'] = 'spawn'
+        elif user.stop_pending:
+            model['pending'] = 'stop'
+        return model
     
     _model_types = {
         'name': str,
@@ -150,7 +156,7 @@ def delete(self, name):
         if user.stop_pending:
             self.set_status(202)
             return
-        if user.spawner is None:
+        if not user.running:
             raise web.HTTPError(400, "%s's server is not running" % name)
         status = yield user.spawner.poll()
         if status is not None:
@@ -175,8 +181,8 @@ def post(self, name):
         user = self.find_user(name)
         if user is None:
             raise web.HTTPError(404)
-        if user.server is None:
-            raise web.HTTPError(400, "%s has no server running" % name)
+        if not user.running:
+            raise web.HTTPError(400, "%s's server is not running" % name)
         self.set_server_cookie(user)
 
 
diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py
--- a/jupyterhub/orm.py
+++ b/jupyterhub/orm.py
@@ -267,6 +267,15 @@ def __repr__(self):
                 name=self.name,
             )
     
+    @property
+    def running(self):
+        """property for whether a user has a running server"""
+        if self.spawner is None:
+            return False
+        if self.server is None:
+            return False
+        return True
+    
     def new_api_token(self):
         """Create a new API token"""
         assert self.id is not None
@@ -285,7 +294,7 @@ def find(cls, db, name):
         Returns None if not found.
         """
         return db.query(cls).filter(cls.name==name).first()
-
+    
     @gen.coroutine
     def spawn(self, spawner_class, base_url='/', hub=None, config=None):
         """Start the user's spawner"""
@@ -298,11 +307,10 @@ def spawn(self, spawner_class, base_url='/', hub=None, config=None):
         )
         db.add(self.server)
         db.commit()
-
+        
         api_token = self.new_api_token()
         db.commit()
         
-        
         spawner = self.spawner = spawner_class(
             config=config,
             user=self,
@@ -314,21 +322,26 @@ def spawn(self, spawner_class, base_url='/', hub=None, config=None):
         spawner.api_token = api_token
         
         self.spawn_pending = True
-        f = spawner.start()
         # wait for spawner.start to return
         try:
+            f = spawner.start()
             yield gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
-        except gen.TimeoutError as e:
-            self.log.warn("{user}'s server failed to start in {s} seconds, giving up".format(
-                user=self.name, s=spawner.start_timeout,
-            ))
+        except Exception as e:
+            if isinstance(e, gen.TimeoutError):
+                self.log.warn("{user}'s server failed to start in {s} seconds, giving up".format(
+                    user=self.name, s=spawner.start_timeout,
+                ))
+            else:
+                self.log.error("Unhandled error starting {user}'s server: {error}".format(
+                    user=self.name, error=e,
+                ))
             try:
                 yield self.stop()
             except Exception:
                 self.log.error("Failed to cleanup {user}'s server that failed to start".format(
                     user=self.name,
                 ), exc_info=True)
-            # raise original TimeoutError
+            # raise original exception
             raise e
         spawner.start_polling()
 
@@ -338,10 +351,15 @@ def spawn(self, spawner_class, base_url='/', hub=None, config=None):
         db.commit()
         try:
             yield self.server.wait_up(http=True)
-        except TimeoutError as e:
-            self.log.warn("{user}'s server never showed up at {url}, giving up".format(
-                user=self.name, url=self.server.url,
-            ))
+        except Exception as e:
+            if isinstance(e, TimeoutError):
+                self.log.warn("{user}'s server never showed up at {url}, giving up".format(
+                    user=self.name, url=self.server.url,
+                ))
+            else:
+                self.log.error("Unhandled error waiting for {user}'s server to show up at {url}: {error}".format(
+                    user=self.name, url=self.server.url, error=e,
+                ))
             try:
                 yield self.stop()
             except Exception:


Test patch:
diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py
--- a/jupyterhub/tests/test_api.py
+++ b/jupyterhub/tests/test_api.py
@@ -104,11 +104,13 @@ def test_get_users(app):
             'name': 'admin',
             'admin': True,
             'server': None,
+            'pending': None,
         },
         {
             'name': 'user',
             'admin': False,
             'server': None,
+            'pending': None,
         }
     ]
 
diff --git a/jupyterhub/tests/test_orm.py b/jupyterhub/tests/test_orm.py
--- a/jupyterhub/tests/test_orm.py
+++ b/jupyterhub/tests/test_orm.py
@@ -3,7 +3,11 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import pytest
+from tornado import gen
+
 from .. import orm
+from .mocking import MockSpawner
 
 
 def test_server(db):
@@ -82,3 +86,20 @@ def test_tokens(db):
     assert found.match(token)
     found = orm.APIToken.find(db, 'something else')
     assert found is None
+
+
+def test_spawn_fails(db, io_loop):
+    user = orm.User(name='aeofel')
+    db.add(user)
+    db.commit()
+    
+    class BadSpawner(MockSpawner):
+        @gen.coroutine
+        def start(self):
+            raise RuntimeError("Split the party")
+    
+    with pytest.raises(Exception) as exc:
+        io_loop.run_sync(lambda : user.spawn(BadSpawner))
+    assert user.server is None
+    assert not user.running
+