Repository: jupyterhub/jupyterhub

Issue:
JupyterHub crashing with invalid database state
It looks like JupyterHub can go into a state where the state of the db is wrong / order of deletion is not right. This causes the hub to never start.

```
singleuser.cmd jupyterhub-singleuser
    Traceback (most recent call last):
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 1623, in launch_instance_async
        yield self.initialize(argv)
      File "/usr/lib/python3.5/types.py", line 179, in throw
        return self.__wrapped.throw(tp, *rest)
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 1385, in initialize
        yield self.init_spawners()
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 1240, in init_spawners
        db.commit()
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 906, in commit
        self.transaction.commit()
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 461, in commit
        self._prepare_impl()
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
        self.session.flush()
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 2171, in flush
        self._flush(objects)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 2291, in _flush
        transaction.rollback(_capture_exception=True)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
        compat.reraise(exc_type, exc_value, exc_tb)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise
        raise value
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 2255, in _flush
        flush_context.execute()
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
        rec.execute(self)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/unitofwork.py", line 577, in execute
        uow
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/persistence.py", line 258, in delete_obj
        cached_connections, mapper, table, delete)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/persistence.py", line 931, in _emit_delete_statements
        c = connection.execute(statement, del_objects)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
        return meth(self, multiparams, params)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
        return connection._execute_clauseelement(self, multiparams, params)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
        compiled_sql, distilled_params
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
        context)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
        exc_info
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
        reraise(type(exception), exception, tb=exc_tb, cause=cause)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 186, in reraise
        raise value.with_traceback(tb)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
        context)
      File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
        cursor.execute(statement, parameters)
    sqlalchemy.exc.IntegrityError: (psycopg2.IntegrityError) update or delete on table "servers" violates foreign key constraint "spawners_server_id_fkey" on table "spawners"
    DETAIL:  Key (id)=(24433) is still referenced from table "spawners".
     [SQL: 'DELETE FROM servers WHERE servers.id = %(id)s'] [parameters: {'id': 24433}]
```


Discussion:
This just happened again :(  First outage of our season. About 6-10 users had this problem, and I had to:

1. Wait for pod to start
2. Wait for it to crash
3. Find id of spawner that crashed
4. Manually remove it from db
Do you have more logs from the incident? I want to see what actions are failing. My guess is that there's a missing or incorrect ondelete.

Resolving patch:
diff --git a/jupyterhub/app.py b/jupyterhub/app.py
--- a/jupyterhub/app.py
+++ b/jupyterhub/app.py
@@ -1219,7 +1219,7 @@ def user_stopped(user, server_name):
                         status = yield spawner.poll()
                     except Exception:
                         self.log.exception("Failed to poll spawner for %s, assuming the spawner is not running.",
-                            user.name if name else '%s|%s' % (user.name, name))
+                            spawner._log_name)
                         status = -1
 
                 if status is None:
@@ -1230,11 +1230,13 @@ def user_stopped(user, server_name):
                     # user not running. This is expected if server is None,
                     # but indicates the user's server died while the Hub wasn't running
                     # if spawner.server is defined.
-                    log = self.log.warning if spawner.server else self.log.debug
-                    log("%s not running.", user.name)
-                    # remove all server or servers entry from db related to the user
                     if spawner.server:
+                        self.log.warning("%s appears to have stopped while the Hub was down", spawner._log_name)
+                        # remove server entry from db
                         db.delete(spawner.orm_spawner.server)
+                        spawner.server = None
+                    else:
+                        self.log.debug("%s not running", spawner._log_name)
             db.commit()
 
             user_summaries.append(_user_summary(user))
diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py
--- a/jupyterhub/orm.py
+++ b/jupyterhub/orm.py
@@ -177,7 +177,7 @@ class Spawner(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'))
 
-    server_id = Column(Integer, ForeignKey('servers.id'))
+    server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
     server = relationship(Server)
 
     state = Column(JSONDict)
@@ -213,7 +213,7 @@ class Service(Base):
     api_tokens = relationship("APIToken", backref="service")
 
     # service-specific interface
-    _server_id = Column(Integer, ForeignKey('servers.id'))
+    _server_id = Column(Integer, ForeignKey('servers.id', ondelete='SET NULL'))
     server = relationship(Server, primaryjoin=_server_id == Server.id)
     pid = Column(Integer)
 
diff --git a/jupyterhub/services/service.py b/jupyterhub/services/service.py
--- a/jupyterhub/services/service.py
+++ b/jupyterhub/services/service.py
@@ -301,5 +301,8 @@ def stop(self):
         if not self.managed:
             raise RuntimeError("Cannot stop unmanaged service %s" % self)
         if self.spawner:
+            if self.orm.server:
+                self.db.delete(self.orm.server)
+                self.db.commit()
             self.spawner.stop_polling()
             return self.spawner.stop()


Test patch:
diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py
--- a/jupyterhub/tests/test_app.py
+++ b/jupyterhub/tests/test_app.py
@@ -8,9 +8,11 @@
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from unittest.mock import patch
 
+from tornado import gen
 import pytest
 
 from .mocking import MockHub
+from .test_api import add_user
 from .. import orm
 from ..app import COOKIE_SECRET_BYTES
 
@@ -161,3 +163,57 @@ def test_load_groups():
     assert gold is not None
     assert sorted([ u.name for u in gold.users ]) == sorted(to_load['gold'])
 
+
+@pytest.mark.gen_test
+def test_resume_spawners(tmpdir, request):
+    if not os.getenv('JUPYTERHUB_TEST_DB_URL'):
+        p = patch.dict(os.environ, {
+            'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s' % tmpdir.join('jupyterhub.sqlite'),
+        })
+        p.start()
+        request.addfinalizer(p.stop)
+    @gen.coroutine
+    def new_hub():
+        app = MockHub()
+        app.config.ConfigurableHTTPProxy.should_start = False
+        yield app.initialize([])
+        return app
+    app = yield new_hub()
+    db = app.db
+    # spawn a user's server
+    name = 'kurt'
+    user = add_user(db, app, name=name)
+    yield user.spawn()
+    proc = user.spawner.proc
+    assert proc is not None
+
+    # stop the Hub without cleaning up servers
+    app.cleanup_servers = False
+    yield app.stop()
+
+    # proc is still running
+    assert proc.poll() is None
+
+    # resume Hub, should still be running
+    app = yield new_hub()
+    db = app.db
+    user = app.users[name]
+    assert user.running
+    assert user.spawner.server is not None
+
+    # stop the Hub without cleaning up servers
+    app.cleanup_servers = False
+    yield app.stop()
+
+    # stop the server while the Hub is down. BAMF!
+    proc.terminate()
+    proc.wait(timeout=10)
+    assert proc.poll() is not None
+
+    # resume Hub, should be stopped
+    app = yield new_hub()
+    db = app.db
+    user = app.users[name]
+    assert not user.running
+    assert user.spawner.server is None
+    assert list(db.query(orm.Server)) == []
diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py
--- a/jupyterhub/tests/test_db.py
+++ b/jupyterhub/tests/test_db.py
@@ -46,4 +46,3 @@ def test_upgrade_entrypoint(tmpdir):
 
     # run tokenapp again, it should work
     tokenapp.start()
-    
\ No newline at end of file
diff --git a/jupyterhub/tests/test_spawner.py b/jupyterhub/tests/test_spawner.py
--- a/jupyterhub/tests/test_spawner.py
+++ b/jupyterhub/tests/test_spawner.py
@@ -299,7 +299,7 @@ def test_spawner_reuse_api_token(db, app):
 
 
 @pytest.mark.gen_test
-def test_spawner_insert_api_token(db, app):
+def test_spawner_insert_api_token(app):
     """Token provided by spawner is not in the db
     
     Insert token into db as a user-provided token.
@@ -326,7 +326,7 @@ def test_spawner_insert_api_token(db, app):
 
 
 @pytest.mark.gen_test
-def test_spawner_bad_api_token(db, app):
+def test_spawner_bad_api_token(app):
     """Tokens are revoked when a Spawner gets another user's token"""
     # we need two users for this one
     user = add_user(app.db, app, name='antimone')
@@ -346,3 +346,37 @@ def test_spawner_bad_api_token(db, app):
         yield user.spawn()
     assert orm.APIToken.find(app.db, other_token) is None
     assert other_user.api_tokens == []
+
+
+@pytest.mark.gen_test
+def test_spawner_delete_server(app):
+    """Test deleting spawner.server
+
+    This can occur during app startup if their server has been deleted.
+    """
+    db = app.db
+    user = add_user(app.db, app, name='gaston')
+    spawner = user.spawner
+    orm_server = orm.Server()
+    db.add(orm_server)
+    db.commit()
+    server_id = orm_server.id
+    spawner.server = Server.from_orm(orm_server)
+    db.commit()
+
+    assert spawner.server is not None
+    assert spawner.orm_spawner.server is not None
+
+    # trigger delete via db
+    db.delete(spawner.orm_spawner.server)
+    db.commit()
+    assert spawner.orm_spawner.server is None
+
+    # setting server = None also triggers delete
+    spawner.server = None
+    db.commit()
+    # verify that the server was actually deleted from the db
+    assert db.query(orm.Server).filter(orm.Server.id == server_id).first() is None
+    # verify that both ORM and top-level references are None
+    assert spawner.orm_spawner.server is None
+    assert spawner.server is None