Repository: jupyterhub/jupyterhub

Issue:
Fewer internal_ssl tests
Currently, we run our internal_ssl tests by re-running several test modules with `ssl_enabled = True`. There's a lot of redundant testing going on here, since only a small subset of these tests are covering additional functionality the second time they run.

I see two ways to improve things:

1. (probably simplest, easiest to keep good coverage) remove the duplicate `internal_ssl_*` test files and switch the ssl_enabled logic to be based on an environment variable, adding ssl testing to the CI matrix. This will significantly reduce the number of tests each run goes through.
2. keep the current always-test-ssl behavior, but opt-in to a much smaller subset of tests known to cover ssl-related functionality:
  - starting, connecting to spawners
  - registering activity
  - services
  - proxy api

I don't have a strong preference for which one, but I think we can speed up the tests quite a bit (times on my laptop):

- with internal_ssl tests: 3:53
- without internal_ssl tests (keeping test_internal_ssl_connections): 2:22

so we are roughly doubling the runtime of all of our test runs to cover internal_ssl configurations.


Discussion:

Resolving patch:
diff --git a/jupyterhub/proxy.py b/jupyterhub/proxy.py
--- a/jupyterhub/proxy.py
+++ b/jupyterhub/proxy.py
@@ -574,6 +574,34 @@ def _remove_pid_file(self):
             self.log.debug("PID file %s already removed", self.pid_file)
             pass
 
+    def _get_ssl_options(self):
+        """List of cmd proxy options to use internal SSL"""
+        cmd = []
+        proxy_api = 'proxy-api'
+        proxy_client = 'proxy-client'
+        api_key = self.app.internal_proxy_certs[proxy_api][
+            'keyfile'
+        ]  # Check content in next test and just patch manulaly or in the config of the file
+        api_cert = self.app.internal_proxy_certs[proxy_api]['certfile']
+        api_ca = self.app.internal_trust_bundles[proxy_api + '-ca']
+
+        client_key = self.app.internal_proxy_certs[proxy_client]['keyfile']
+        client_cert = self.app.internal_proxy_certs[proxy_client]['certfile']
+        client_ca = self.app.internal_trust_bundles[proxy_client + '-ca']
+
+        cmd.extend(['--api-ssl-key', api_key])
+        cmd.extend(['--api-ssl-cert', api_cert])
+        cmd.extend(['--api-ssl-ca', api_ca])
+        cmd.extend(['--api-ssl-request-cert'])
+        cmd.extend(['--api-ssl-reject-unauthorized'])
+
+        cmd.extend(['--client-ssl-key', client_key])
+        cmd.extend(['--client-ssl-cert', client_cert])
+        cmd.extend(['--client-ssl-ca', client_ca])
+        cmd.extend(['--client-ssl-request-cert'])
+        cmd.extend(['--client-ssl-reject-unauthorized'])
+        return cmd
+
     async def start(self):
         """Start the proxy process"""
         # check if there is a previous instance still around
@@ -605,27 +633,7 @@ async def start(self):
         if self.ssl_cert:
             cmd.extend(['--ssl-cert', self.ssl_cert])
         if self.app.internal_ssl:
-            proxy_api = 'proxy-api'
-            proxy_client = 'proxy-client'
-            api_key = self.app.internal_proxy_certs[proxy_api]['keyfile']
-            api_cert = self.app.internal_proxy_certs[proxy_api]['certfile']
-            api_ca = self.app.internal_trust_bundles[proxy_api + '-ca']
-
-            client_key = self.app.internal_proxy_certs[proxy_client]['keyfile']
-            client_cert = self.app.internal_proxy_certs[proxy_client]['certfile']
-            client_ca = self.app.internal_trust_bundles[proxy_client + '-ca']
-
-            cmd.extend(['--api-ssl-key', api_key])
-            cmd.extend(['--api-ssl-cert', api_cert])
-            cmd.extend(['--api-ssl-ca', api_ca])
-            cmd.extend(['--api-ssl-request-cert'])
-            cmd.extend(['--api-ssl-reject-unauthorized'])
-
-            cmd.extend(['--client-ssl-key', client_key])
-            cmd.extend(['--client-ssl-cert', client_cert])
-            cmd.extend(['--client-ssl-ca', client_ca])
-            cmd.extend(['--client-ssl-request-cert'])
-            cmd.extend(['--client-ssl-reject-unauthorized'])
+            cmd.extend(self._get_ssl_options())
         if self.app.statsd_host:
             cmd.extend(
                 [


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -77,6 +77,10 @@ jobs:
         #   Tests everything when the user instances are started with
         #   jupyter_server instead of notebook.
         #
+        # ssl:
+        #   Tests everything using internal SSL connections instead of
+        #   unencrypted HTTP
+        #
         # main_dependencies:
         #   Tests everything when the we use the latest available dependencies
         #   from: ipytraitlets.
@@ -89,6 +93,8 @@ jobs:
             subdomain: subdomain
           - python: "3.7"
             db: mysql
+          - python: "3.7"
+            ssl: ssl
           - python: "3.8"
             db: postgres
           - python: "3.8"
@@ -109,6 +115,9 @@ jobs:
               echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV
               echo "JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:3306/jupyterhub" >> $GITHUB_ENV
           fi
+          if [ "${{ matrix.ssl }}" == "ssl" ]; then
+              echo "SSL_ENABLED=1" >> $GITHUB_ENV
+          fi
           if [ "${{ matrix.db }}" == "postgres" ]; then
               echo "PGHOST=127.0.0.1" >> $GITHUB_ENV
               echo "PGUSER=test_user" >> $GITHUB_ENV
diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py
--- a/jupyterhub/tests/conftest.py
+++ b/jupyterhub/tests/conftest.py
@@ -74,7 +74,9 @@ def ssl_tmpdir(tmpdir_factory):
 def app(request, io_loop, ssl_tmpdir):
     """Mock a jupyterhub app for testing"""
     mocked_app = None
-    ssl_enabled = getattr(request.module, "ssl_enabled", False)
+    ssl_enabled = getattr(
+        request.module, 'ssl_enabled', os.environ.get('SSL_ENABLED', False)
+    )
     kwargs = dict()
     if ssl_enabled:
         kwargs.update(dict(internal_ssl=True, internal_certs_location=str(ssl_tmpdir)))
diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py
--- a/jupyterhub/tests/test_api.py
+++ b/jupyterhub/tests/test_api.py
@@ -27,7 +27,6 @@
 from .utils import auth_header
 from .utils import find_user
 
-
 # --------------------
 # Authentication tests
 # --------------------
diff --git a/jupyterhub/tests/test_internal_ssl_api.py b/jupyterhub/tests/test_internal_ssl_api.py
deleted file mode 100644
--- a/jupyterhub/tests/test_internal_ssl_api.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""Tests for the SSL enabled REST API."""
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-from jupyterhub.tests.test_api import *
-
-ssl_enabled = True
diff --git a/jupyterhub/tests/test_internal_ssl_app.py b/jupyterhub/tests/test_internal_ssl_app.py
deleted file mode 100644
--- a/jupyterhub/tests/test_internal_ssl_app.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""Test the JupyterHub entry point with internal ssl"""
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-import jupyterhub.tests.mocking
-from jupyterhub.tests.test_app import *
-
-ssl_enabled = True
diff --git a/jupyterhub/tests/test_internal_ssl_spawner.py b/jupyterhub/tests/test_internal_ssl_spawner.py
deleted file mode 100644
--- a/jupyterhub/tests/test_internal_ssl_spawner.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""Tests for process spawning with internal_ssl"""
-# Copyright (c) Jupyter Development Team.
-# Distributed under the terms of the Modified BSD License.
-from jupyterhub.tests.test_spawner import *
-
-ssl_enabled = True
diff --git a/jupyterhub/tests/test_proxy.py b/jupyterhub/tests/test_proxy.py
--- a/jupyterhub/tests/test_proxy.py
+++ b/jupyterhub/tests/test_proxy.py
@@ -9,12 +9,12 @@
 import pytest
 from traitlets.config import Config
 
-from .. import orm
 from ..utils import url_path_join as ujoin
 from ..utils import wait_for_http_server
 from .mocking import MockHub
 from .test_api import add_user
 from .test_api import api_request
+from .utils import skip_if_ssl
 
 
 @pytest.fixture
@@ -27,6 +27,7 @@ def disable_check_routes(app):
         app.last_activity_callback.start()
 
 
+@skip_if_ssl
 async def test_external_proxy(request):
     auth_token = 'secret!'
     proxy_ip = '127.0.0.1'
diff --git a/jupyterhub/tests/test_services.py b/jupyterhub/tests/test_services.py
--- a/jupyterhub/tests/test_services.py
+++ b/jupyterhub/tests/test_services.py
@@ -17,6 +17,7 @@
 from ..utils import wait_for_http_server
 from .mocking import public_url
 from .utils import async_requests
+from .utils import skip_if_ssl
 
 mockservice_path = os.path.dirname(os.path.abspath(__file__))
 mockservice_py = os.path.join(mockservice_path, 'mockservice.py')
@@ -63,6 +64,7 @@ async def test_managed_service(mockservice):
     assert service.proc.poll() is None
 
 
+@skip_if_ssl
 async def test_proxy_service(app, mockservice_url):
     service = mockservice_url
     name = service.name
@@ -76,6 +78,7 @@ async def test_proxy_service(app, mockservice_url):
     assert r.text.endswith(path)
 
 
+@skip_if_ssl
 async def test_external_service(app):
     name = 'external'
     async with external_service(app, name=name) as env:
diff --git a/jupyterhub/tests/test_services_auth.py b/jupyterhub/tests/test_services_auth.py
--- a/jupyterhub/tests/test_services_auth.py
+++ b/jupyterhub/tests/test_services_auth.py
@@ -35,6 +35,7 @@
 
 # mock for sending monotonic counter way into the future
 monotonic_future = mock.patch('time.monotonic', lambda: sys.maxsize)
+ssl_enabled = False
 
 
 def test_expiring_dict():
diff --git a/jupyterhub/tests/utils.py b/jupyterhub/tests/utils.py
--- a/jupyterhub/tests/utils.py
+++ b/jupyterhub/tests/utils.py
@@ -1,6 +1,8 @@
 import asyncio
+import os
 from concurrent.futures import ThreadPoolExecutor
 
+import pytest
 import requests
 from certipy import Certipy
 
@@ -52,6 +54,12 @@ def ssl_setup(cert_dir, authority_name):
     return external_certs
 
 
+"""Skip tests that don't work under internal-ssl when testing under internal-ssl"""
+skip_if_ssl = pytest.mark.skipif(
+    os.environ.get('SSL_ENABLED', False), reason="Does not use internal SSL"
+)
+
+
 def check_db_locks(func):
     """Decorator that verifies no locks are held on database upon exit.