Repository: jupyterhub/jupyterhub

Issue:
Error when running jupyterhub upgrade-db
**How to reproduce the issue**
Run `jupyterhub upgrade-db --db=mysql+pymysql://db_url_xxx`

**What you expected to happen**
DB is upgraded for 0.8

**What actually happens**
```
INFO  [alembic.runtime.migration] Context impl MySQLImpl.
INFO  [alembic.runtime.migration] Will assume non-transactional DDL.
INFO  [alembic.runtime.migration] Running upgrade  -> 19c0846f6344, base revision for 0.5
INFO  [alembic.runtime.migration] Running upgrade 19c0846f6344 -> eeb276e51423, auth_state
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 166, in execute
    result = self._query(query)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 322, in _query
    conn.query(q)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 856, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1057, in _read_query_result
    result.read()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1340, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1014, in _read_packet
    packet.check_error()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 393, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
pymysql.err.InternalError: (1060, "Duplicate column name 'auth_state'")

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/bin/alembic", line 11, in <module>
    load_entry_point('alembic==0.9.5', 'console_scripts', 'alembic')()
  File "/usr/local/lib/python3.5/dist-packages/alembic/config.py", line 479, in main
    CommandLine(prog=prog).main(argv=argv)
  File "/usr/local/lib/python3.5/dist-packages/alembic/config.py", line 473, in main
    self.run_cmd(cfg, options)
  File "/usr/local/lib/python3.5/dist-packages/alembic/config.py", line 456, in run_cmd
    **dict((k, getattr(options, k, None)) for k in kwarg)
  File "/usr/local/lib/python3.5/dist-packages/alembic/command.py", line 254, in upgrade
    script.run_env()
  File "/usr/local/lib/python3.5/dist-packages/alembic/script/base.py", line 425, in run_env
    util.load_python_file(self.dir, 'env.py')
  File "/usr/local/lib/python3.5/dist-packages/alembic/util/pyfiles.py", line 93, in load_python_file
    module = load_module_py(module_id, path)
  File "/usr/local/lib/python3.5/dist-packages/alembic/util/compat.py", line 64, in load_module_py
    module_id, path).load_module(module_id)
  File "<frozen importlib._bootstrap_external>", line 396, in _check_name_wrapper
  File "<frozen importlib._bootstrap_external>", line 817, in load_module
  File "<frozen importlib._bootstrap_external>", line 676, in load_module
  File "<frozen importlib._bootstrap>", line 268, in _load_module_shim
  File "<frozen importlib._bootstrap>", line 693, in _load
  File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 673, in exec_module
  File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
  File "/usr/local/lib/python3.5/dist-packages/jupyterhub/alembic/env.py", line 70, in <module>
    run_migrations_online()
  File "/usr/local/lib/python3.5/dist-packages/jupyterhub/alembic/env.py", line 65, in run_migrations_online
    context.run_migrations()
  File "<string>", line 8, in run_migrations
  File "/usr/local/lib/python3.5/dist-packages/alembic/runtime/environment.py", line 836, in run_migrations
    self.get_context().run_migrations(**kw)
  File "/usr/local/lib/python3.5/dist-packages/alembic/runtime/migration.py", line 330, in run_migrations
    step.migration_fn(**kw)
  File "/usr/local/lib/python3.5/dist-packages/jupyterhub/alembic/versions/eeb276e51423_auth_state.py", line 21, in upgrade
    op.add_column('users', sa.Column('auth_state', JSONDict))
  File "<string>", line 8, in add_column
  File "<string>", line 3, in add_column
  File "/usr/local/lib/python3.5/dist-packages/alembic/operations/ops.py", line 1565, in add_column
    return operations.invoke(op)
  File "/usr/local/lib/python3.5/dist-packages/alembic/operations/base.py", line 318, in invoke
    return fn(self, operation)
  File "/usr/local/lib/python3.5/dist-packages/alembic/operations/toimpl.py", line 123, in add_column
    schema=schema
  File "/usr/local/lib/python3.5/dist-packages/alembic/ddl/impl.py", line 172, in add_column
    self._exec(base.AddColumn(table_name, column, schema=schema))
  File "/usr/local/lib/python3.5/dist-packages/alembic/ddl/impl.py", line 118, in _exec
    return conn.execute(construct, *multiparams, **params)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
    return connection._execute_ddl(self, multiparams, params)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
    compiled
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
    context)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
    exc_info
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 186, in reraise
    raise value.with_traceback(tb)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 166, in execute
    result = self._query(query)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 322, in _query
    conn.query(q)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 856, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1057, in _read_query_result
    result.read()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1340, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1014, in _read_packet
    packet.check_error()
  File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 393, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/local/lib/python3.5/dist-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1060, "Duplicate column name 'auth_state'") [SQL: 'ALTER TABLE users ADD COLUMN auth_state TEXT']
[E 2017-09-25 21:15:42.980 JupyterHub app:1527]
    Traceback (most recent call last):
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 1525, in launch_instance_async
        yield self.start()
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 1436, in start
        self.subapp.start()
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/app.py", line 185, in start
        dbutil.upgrade(hub.db_url)
      File "/usr/local/lib/python3.5/dist-packages/jupyterhub/dbutil.py", line 80, in upgrade
        ['alembic', '-c', alembic_ini, 'upgrade', revision]
      File "/usr/lib/python3.5/subprocess.py", line 271, in check_call
        raise CalledProcessError(retcode, cmd)
    subprocess.CalledProcessError: Command '['alembic', '-c', '/tmp/tmptsm58x_8/alembic.ini', 'upgrade', 'head']' returned non-zero exit status 1
```
**Share what version of JupyterHub you are using**
Jupyterhub 0.7.2 ->0.8.0rc2

Running `jupyter troubleshoot` from the command line, if possible, and posting
its output would also be helpful.

```
$PATH:
	/usr/local/bin
	/usr/local/sbin
	/usr/local/bin
	/usr/sbin
	/usr/bin
	/sbin
	/bin

sys.path:
	/usr/local/bin
	/srv/jupyterhub
	/usr/lib/python35.zip
	/usr/lib/python3.5
	/usr/lib/python3.5/plat-x86_64-linux-gnu
	/usr/lib/python3.5/lib-dynload
	/usr/local/lib/python3.5/dist-packages
	/usr/lib/python3/dist-packages

sys.executable:
	/usr/bin/python3

sys.version:
	3.5.3 (default, Jan 19 2017, 14:11:04)
	[GCC 6.3.0 20170118]

platform.platform():
	Linux-4.4.52+-x86_64-with-Ubuntu-17.04-zesty

which -a jupyter:
	/usr/local/bin/jupyter
	/usr/local/bin/jupyter
```



Discussion:
same error as #1162 which is closed 
It probably can be solved by call check_db_revision before dbutil.upgrade(hub.db_url).
Got another error when running `jupyterhub upgrade-db --db=mysql+pymysql://db_url_xxx` after executed below sql to set alembic_version.
```
CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL);
INSERT INTO alembic_version (version_num) VALUES ('af4cbdb2d13c');
```

```
[I 2017-09-26 10:29:41.865 alembic.runtime.migration migration:117] Context impl MySQLImpl.
[I 2017-09-26 10:29:41.866 alembic.runtime.migration migration:122] Will assume non-transactional DDL.
[I 2017-09-26 10:29:42.060 alembic.runtime.migration migration:327] Running upgrade af4cbdb2d13c -> 3ec6993fe20c, 0.8 changes
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 504, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python3.6/site-packages/pymysql/cursors.py", line 166, in execute
    result = self._query(query)
  File "/usr/local/lib/python3.6/site-packages/pymysql/cursors.py", line 322, in _query
    conn.query(q)
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 856, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1057, in _read_query_result
    result.read()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1340, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1014, in _read_packet
    packet.check_error()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 393, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/local/lib/python3.6/site-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
pymysql.err.InternalError: (1553, "Cannot drop index '_server_id': needed in a foreign key constraint")

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/bin/alembic", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.6/site-packages/alembic/config.py", line 479, in main
    CommandLine(prog=prog).main(argv=argv)
  File "/usr/local/lib/python3.6/site-packages/alembic/config.py", line 473, in main
    self.run_cmd(cfg, options)
  File "/usr/local/lib/python3.6/site-packages/alembic/config.py", line 456, in run_cmd
    **dict((k, getattr(options, k, None)) for k in kwarg)
  File "/usr/local/lib/python3.6/site-packages/alembic/command.py", line 254, in upgrade
    script.run_env()
  File "/usr/local/lib/python3.6/site-packages/alembic/script/base.py", line 425, in run_env
    util.load_python_file(self.dir, 'env.py')
  File "/usr/local/lib/python3.6/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
    module = load_module_py(module_id, path)
  File "/usr/local/lib/python3.6/site-packages/alembic/util/compat.py", line 64, in load_module_py
    module_id, path).load_module(module_id)
  File "<frozen importlib._bootstrap_external>", line 399, in _check_name_wrapper
  File "<frozen importlib._bootstrap_external>", line 823, in load_module
  File "<frozen importlib._bootstrap_external>", line 682, in load_module
  File "<frozen importlib._bootstrap>", line 251, in _load_module_shim
  File "<frozen importlib._bootstrap>", line 675, in _load
  File "<frozen importlib._bootstrap>", line 655, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 678, in exec_module
  File "<frozen importlib._bootstrap>", line 205, in _call_with_frames_removed
  File "/usr/local/lib/python3.6/site-packages/jupyterhub/alembic/env.py", line 89, in <module>
    run_migrations_online()
  File "/usr/local/lib/python3.6/site-packages/jupyterhub/alembic/env.py", line 84, in run_migrations_online
    context.run_migrations()
  File "<string>", line 8, in run_migrations
  File "/usr/local/lib/python3.6/site-packages/alembic/runtime/environment.py", line 836, in run_migrations
    self.get_context().run_migrations(**kw)
  File "/usr/local/lib/python3.6/site-packages/alembic/runtime/migration.py", line 330, in run_migrations
    step.migration_fn(**kw)
  File "/usr/local/lib/python3.6/site-packages/jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py", line 39, in upgrade
    op.drop_column('users', '_server_id')
  File "<string>", line 8, in drop_column
  File "<string>", line 3, in drop_column
  File "/usr/local/lib/python3.6/site-packages/alembic/operations/ops.py", line 1667, in drop_column
    return operations.invoke(op)
  File "/usr/local/lib/python3.6/site-packages/alembic/operations/base.py", line 318, in invoke
    return fn(self, operation)
  File "/usr/local/lib/python3.6/site-packages/alembic/operations/toimpl.py", line 81, in drop_column
    **operation.kw
  File "/usr/local/lib/python3.6/site-packages/alembic/ddl/impl.py", line 175, in drop_column
    self._exec(base.DropColumn(table_name, column, schema=schema))
  File "/usr/local/lib/python3.6/site-packages/alembic/ddl/impl.py", line 118, in _exec
    return conn.execute(construct, *multiparams, **params)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 945, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
    return connection._execute_ddl(self, multiparams, params)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
    compiled
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
    context)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
    exc_info
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 186, in reraise
    raise value.with_traceback(tb)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 504, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python3.6/site-packages/pymysql/cursors.py", line 166, in execute
    result = self._query(query)
  File "/usr/local/lib/python3.6/site-packages/pymysql/cursors.py", line 322, in _query
    conn.query(q)
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 856, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1057, in _read_query_result
    result.read()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1340, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 1014, in _read_packet
    packet.check_error()
  File "/usr/local/lib/python3.6/site-packages/pymysql/connections.py", line 393, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/local/lib/python3.6/site-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1553, "Cannot drop index '_server_id': needed in a foreign key constraint") [SQL: 'ALTER TABLE users DROP COLUMN _server_id']
[E 2017-09-26 10:29:44.002 JupyterHub app:1626]
    Traceback (most recent call last):
      File "/usr/local/lib/python3.6/site-packages/jupyterhub/app.py", line 1624, in launch_instance_async
        yield self.start()
      File "/usr/local/lib/python3.6/site-packages/jupyterhub/app.py", line 1514, in start
        self.subapp.start()
      File "/usr/local/lib/python3.6/site-packages/jupyterhub/app.py", line 192, in start
        dbutil.upgrade(hub.db_url)
      File "/usr/local/lib/python3.6/site-packages/jupyterhub/dbutil.py", line 83, in upgrade
        ['alembic', '-c', alembic_ini, 'upgrade', revision]
      File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 291, in check_call
        raise CalledProcessError(retcode, cmd)
    subprocess.CalledProcessError: Command '['alembic', '-c', '/var/folders/vv/szpdwz315wd9z0h6gvw4cqcm0000gq/T/tmp5u3sy1q7/alembic.ini', 'upgrade', 'head']' returned non-zero exit status 1.
```
Here are SQL need to run before upgrading:
```
CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL);
INSERT INTO alembic_version (version_num) VALUES ('af4cbdb2d13c');
ALTER TABLE users DROP FOREIGN KEY users_ibfk_1;
```

Resolving patch:
diff --git a/jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py b/jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py
--- a/jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py
+++ b/jupyterhub/alembic/versions/3ec6993fe20c_encrypted_auth_state.py
@@ -36,6 +36,10 @@ def upgrade():
     # drop some columns no longer in use
     try:
         op.drop_column('users', 'auth_state')
+        # mysql cannot drop _server_id without also dropping
+        # implicitly created foreign key
+        if op.get_context().dialect.name == 'mysql':
+            op.drop_constraint('users_ibfk_1', 'users',  type_='foreignkey')
         op.drop_column('users', '_server_id')
     except sa.exc.OperationalError:
         # this won't be a problem moving forward, but downgrade will fail
diff --git a/jupyterhub/app.py b/jupyterhub/app.py
--- a/jupyterhub/app.py
+++ b/jupyterhub/app.py
@@ -23,6 +23,7 @@
 
 from jinja2 import Environment, FileSystemLoader
 
+from sqlalchemy import create_engine
 from sqlalchemy.exc import OperationalError
 
 from tornado.httpclient import AsyncHTTPClient
@@ -189,6 +190,13 @@ def start(self):
             db_file = hub.db_url.split(':///', 1)[1]
             self._backup_db_file(db_file)
         self.log.info("Upgrading %s", hub.db_url)
+        # run check-db-revision first
+        engine = create_engine(hub.db_url)
+        try:
+            orm.check_db_revision(engine)
+        except orm.DatabaseSchemaMismatch:
+            # ignore mismatch error because that's what we are here for!
+            pass
         dbutil.upgrade(hub.db_url)
 
 


Test patch:
diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py
--- a/jupyterhub/tests/test_db.py
+++ b/jupyterhub/tests/test_db.py
@@ -4,6 +4,7 @@
 
 import pytest
 from pytest import raises
+from traitlets.config import Config
 
 from ..dbutil import upgrade
 from ..app import NewToken, UpgradeDB, JupyterHub
@@ -21,28 +22,35 @@ def generate_old_db(path):
 def test_upgrade(tmpdir):
     print(tmpdir)
     db_url = generate_old_db(str(tmpdir))
-    print(db_url)
     upgrade(db_url)
 
 @pytest.mark.gen_test
 def test_upgrade_entrypoint(tmpdir):
-    generate_old_db(str(tmpdir))
+    db_url = os.getenv('JUPYTERHUB_TEST_UPGRADE_DB_URL')
+    if not db_url:
+        # default: sqlite
+        db_url = generate_old_db(str(tmpdir))
+    cfg = Config()
+    cfg.JupyterHub.db_url = db_url
+
     tmpdir.chdir()
-    tokenapp = NewToken()
+    tokenapp = NewToken(config=cfg)
     tokenapp.initialize(['kaylee'])
     with raises(SystemExit):
         tokenapp.start()
 
-    sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*'))
-    assert len(sqlite_files) == 1
+    if 'sqlite' in db_url:
+        sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*'))
+        assert len(sqlite_files) == 1
 
-    upgradeapp = UpgradeDB()
+    upgradeapp = UpgradeDB(config=cfg)
     yield upgradeapp.initialize([])
     upgradeapp.start()
 
     # check that backup was created:
-    sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*'))
-    assert len(sqlite_files) == 2
+    if 'sqlite' in db_url:
+        sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*'))
+        assert len(sqlite_files) == 2
 
     # run tokenapp again, it should work
     tokenapp.start()