Repository: jupyterhub/jupyterhub

Issue:
Running jupyterhub upgrade-db with PostgreSQL database fails
**How to reproduce the issue**

Run `jupyterhub upgrade-db` with a PostgreSQL database to upgrade to 99a28a4418e1.

**What you expected to happen**

Successful schema update.

**What actually happens**

It fails with an sqlalchemy `ProgrammingError` message that originates here:

https://github.com/jupyterhub/jupyterhub/blob/master/jupyterhub/alembic/versions/99a28a4418e1_user_created.py#L40

in particular I think that should be `IS NOT NULL` not just `NOT NULL`.  I substituted this live and it allowed the upgrade to proceed.

**Share what version of JupyterHub you are using**

Latest master.


Discussion:

Resolving patch:
diff --git a/jupyterhub/alembic/versions/56cc5a70207e_token_tracking.py b/jupyterhub/alembic/versions/56cc5a70207e_token_tracking.py
--- a/jupyterhub/alembic/versions/56cc5a70207e_token_tracking.py
+++ b/jupyterhub/alembic/versions/56cc5a70207e_token_tracking.py
@@ -15,6 +15,9 @@
 from alembic import op
 import sqlalchemy as sa
 
+import logging
+logger = logging.getLogger('alembic')
+
 
 def upgrade():
     tables = op.get_bind().engine.table_names()
@@ -24,8 +27,11 @@ def upgrade():
     if 'oauth_access_tokens' in tables:
         op.add_column('oauth_access_tokens', sa.Column('created', sa.DateTime(), nullable=True))
         op.add_column('oauth_access_tokens', sa.Column('last_activity', sa.DateTime(), nullable=True))
-        op.create_foreign_key(None, 'oauth_access_tokens', 'oauth_clients', ['client_id'], ['identifier'], ondelete='CASCADE')
-        op.create_foreign_key(None, 'oauth_codes', 'oauth_clients', ['client_id'], ['identifier'], ondelete='CASCADE')
+        if op.get_context().dialect.name == 'sqlite':
+            logger.warning("sqlite cannot use ALTER TABLE to create foreign keys. Upgrade will be incomplete.")
+        else:
+            op.create_foreign_key(None, 'oauth_access_tokens', 'oauth_clients', ['client_id'], ['identifier'], ondelete='CASCADE')
+            op.create_foreign_key(None, 'oauth_codes', 'oauth_clients', ['client_id'], ['identifier'], ondelete='CASCADE')
 
 
 def downgrade():
diff --git a/jupyterhub/alembic/versions/99a28a4418e1_user_created.py b/jupyterhub/alembic/versions/99a28a4418e1_user_created.py
--- a/jupyterhub/alembic/versions/99a28a4418e1_user_created.py
+++ b/jupyterhub/alembic/versions/99a28a4418e1_user_created.py
@@ -37,7 +37,7 @@ def upgrade():
         c.execute("""
             UPDATE spawners
             SET started='%s'
-            WHERE server_id NOT NULL
+            WHERE server_id IS NOT NULL
             """ % (now,)
         )
 


Test patch:
diff --git a/jupyterhub/tests/old-jupyterhub.sqlite b/jupyterhub/tests/old-jupyterhub.sqlite
deleted file mode 100644
Binary files a/jupyterhub/tests/old-jupyterhub.sqlite and /dev/null differ
diff --git a/jupyterhub/tests/populate_db.py b/jupyterhub/tests/populate_db.py
new file mode 100644
--- /dev/null
+++ b/jupyterhub/tests/populate_db.py
@@ -0,0 +1,98 @@
+"""script to populate a jupyterhub database
+
+Run with old versions of jupyterhub to test upgrade/downgrade
+
+used in test_db.py
+"""
+
+from datetime import datetime
+import os
+
+import jupyterhub
+from jupyterhub import orm
+
+
+def populate_db(url):
+    """Populate a jupyterhub database"""
+    db = orm.new_session_factory(url)()
+    # create some users
+    admin = orm.User(name='admin', admin=True)
+    db.add(admin)
+    user = orm.User(name='has-server')
+    db.add(user)
+    db.commit()
+
+    # create a group
+    g = orm.Group(name='group')
+    db.add(g)
+    db.commit()
+    g.users.append(user)
+    db.commit()
+
+    service = orm.Service(name='service')
+    db.add(service)
+    db.commit()
+
+    # create some API tokens
+    user.new_api_token()
+    admin.new_api_token()
+
+    # services got API tokens in 0.7
+    if jupyterhub.version_info >= (0, 7):
+        # create a group
+        service.new_api_token()
+
+    # Create a Spawner for user
+    if jupyterhub.version_info >= (0, 8):
+        # create spawner for user
+        spawner = orm.Spawner(name='', user=user)
+        db.add(spawner)
+        db.commit()
+        spawner.server = orm.Server()
+        db.commit()
+
+        # admin's spawner is not running
+        spawner = orm.Spawner(name='', user=admin)
+        db.add(spawner)
+        db.commit()
+    else:
+        user.server = orm.Server()
+        db.commit()
+
+    # create some oauth objects
+    if jupyterhub.version_info >= (0, 8):
+        # create oauth client
+        client = orm.OAuthClient(identifier='oauth-client')
+        db.add(client)
+        db.commit()
+        code = orm.OAuthCode(client_id=client.identifier)
+        db.add(code)
+        db.commit()
+        access_token = orm.OAuthAccessToken(
+            client_id=client.identifier,
+            user_id=user.id,
+            grant_type=orm.GrantType.authorization_code,
+        )
+        db.add(access_token)
+        db.commit()
+
+    # set some timestamps added in 0.9
+    if jupyterhub.version_info >= (0, 9):
+        assert user.created
+        assert admin.created
+        # set last_activity
+        user.last_activity = datetime.utcnow()
+        spawner = user.orm_spawners['']
+        spawner.started = datetime.utcnow()
+        spawner.last_activity = datetime.utcnow()
+        db.commit()
+
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) > 1:
+        url = sys.argv[1]
+    else:
+        url = 'sqlite:///jupyterhub.sqlite'
+
+    populate_db(url)
diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py
--- a/jupyterhub/tests/test_db.py
+++ b/jupyterhub/tests/test_db.py
@@ -1,45 +1,70 @@
 from glob import glob
 import os
-import shutil
+from subprocess import check_call
+import sys
+import tempfile
 
 import pytest
 from pytest import raises
 from traitlets.config import Config
 
-from ..dbutil import upgrade
 from ..app import NewToken, UpgradeDB, JupyterHub
 
 
-here = os.path.dirname(__file__)
-old_db = os.path.join(here, 'old-jupyterhub.sqlite')
+here = os.path.abspath(os.path.dirname(__file__))
+populate_db = os.path.join(here, 'populate_db.py')
 
-def generate_old_db(path):
-    db_path = os.path.join(path, "jupyterhub.sqlite")
-    print(old_db, db_path)
-    shutil.copy(old_db, db_path)
-    return 'sqlite:///%s' % db_path
 
-def test_upgrade(tmpdir):
-    print(tmpdir)
-    db_url = generate_old_db(str(tmpdir))
-    upgrade(db_url)
+def generate_old_db(env_dir, hub_version, db_url):
+    """Generate an old jupyterhub database
 
+    Installs a particular jupyterhub version in a virtualenv
+    and runs populate_db.py to populate a database
+    """
+    env_pip = os.path.join(env_dir, 'bin', 'pip')
+    env_py = os.path.join(env_dir, 'bin', 'python')
+    check_call([sys.executable, '-m', 'virtualenv', env_dir])
+    pkgs = ['jupyterhub==' + hub_version]
+    if 'mysql' in db_url:
+        pkgs.append('mysql-connector<2.2')
+    elif 'postgres' in db_url:
+        pkgs.append('psycopg2')
+    check_call([env_pip, 'install'] + pkgs)
+    check_call([env_py, populate_db, db_url])
+
+
+@pytest.mark.parametrize(
+    'hub_version',
+    [
+        '0.7.2',
+        '0.8.1',
+    ],
+)
 @pytest.mark.gen_test
-def test_upgrade_entrypoint(tmpdir):
-    db_url = os.getenv('JUPYTERHUB_TEST_UPGRADE_DB_URL')
-    if not db_url:
-        # default: sqlite
-        db_url = generate_old_db(str(tmpdir))
+def test_upgrade(tmpdir, hub_version):
+    db_url = os.getenv('JUPYTERHUB_TEST_DB_URL')
+    if db_url:
+        db_url += '_upgrade_' + hub_version.replace('.', '')
+    else:
+        db_url = 'sqlite:///jupyterhub.sqlite'
+    tmpdir.chdir()
+
+    # use persistent temp env directory
+    # to reuse across multiple runs
+    env_dir = os.path.join(tempfile.gettempdir(), 'test-hub-upgrade-%s' % hub_version)
+
+    generate_old_db(env_dir, hub_version, db_url)
+
     cfg = Config()
     cfg.JupyterHub.db_url = db_url
 
-    tmpdir.chdir()
     tokenapp = NewToken(config=cfg)
-    tokenapp.initialize(['kaylee'])
+    tokenapp.initialize(['admin'])
     with raises(SystemExit):
         tokenapp.start()
 
     if 'sqlite' in db_url:
+        fname = db_url.split(':///')[1]
         sqlite_files = glob(os.path.join(str(tmpdir), 'jupyterhub.sqlite*'))
         assert len(sqlite_files) == 1