Repository: jupyterhub/zero-to-jupyterhub-k8s

Issue:
PR Discussion - chartpress to not rebuild all the time
As @clkao has realized and pointed out, chartpress is rebuilding all the used images all the time during testing. I'm right now looking at a build job (https://travis-ci.org/jupyterhub/zero-to-jupyterhub-k8s/jobs/583123527) where no image has been changed, but they are all being rebuilt. Sometimes the Travis workers are relatively busy and this end up taking even than 20 minutes to complete.

While there is multiple improvement possibilities, an essential one I think is to not rebuild all the time.

---

This was a duplicated issue of this: https://github.com/jupyterhub/chartpress/issues/40
multi-stage CI
While working on #1259, i found that every PR's env matrix is rebuilding all images using chartpress.  To make things run faster, we should probably use [multi-stage travis](https://docs.travis-ci.com/user/build-stages/) to do the following:

1. tools/templates/lint-and-validate.py
2. chartpress and publish
3. setup k8s env and run tests

Note that only stage 3 should do matrix expansion into multiple jobs for different k8s version/config. and it needs to run chartpress render without building the image, since all stage 3 jobs can pull images built by stage 2.

For stage 2, we might be able to pull images before the current COMMIT_SHA, and this should avoid doing unnecessary image builds if nothing changed in the image subdirectories.


Discussion:

Resolving patch:
diff --git a/ci/kind-load-docker-images.py b/ci/kind-load-docker-images.py
new file mode 100755
--- /dev/null
+++ b/ci/kind-load-docker-images.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""
+Functions using kind to load docker images.
+
+Run `kind load docker-image <image:tag>` on all the docker images within
+values.yaml that is available locally on the host as first verified with `docker
+images --quiet <image:tag>`. If we could capture this directly from chartpress
+build output it would be quicker.
+
+Example: to get help about the parameters and their default values.
+
+    ci/kind-load-docker-images.py --help
+
+Example: after you have run `chartpress` to modify the values.yaml files with
+the newly built image name and tags, you can run this command to ensure the kind
+cluster named "dev" gets the required docker images from your local registry.
+
+    ci/kind-load-docker-images.py --kind-cluster dev
+"""
+
+import sys
+import argparse
+import pipes
+import subprocess
+
+import yaml
+
+
+def check_output(cmd, **kwargs):
+    """Run a subcommand and exit if it fails"""
+    try:
+        return subprocess.check_output(cmd, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print(
+            "`{}` exited with status {}".format(
+                " ".join(map(pipes.quote, cmd)), e.returncode
+            ),
+            file=sys.stderr,
+        )
+        sys.exit(e.returncode)
+
+
+def get_element_from_path(path, dictionary):
+    keys = path.split(".")
+    e = dictionary
+    for key in keys:
+        e = e[key]
+    return e
+
+
+def extract_images_from_values(chartpress_file, values_file):
+    """Returns a list of image:tag strings given a values.yaml file."""
+
+    with open(chartpress_file) as f:
+        chartpress = yaml.safe_load(f)
+
+    with open(values_file) as f:
+        values = yaml.safe_load(f)
+
+    image_paths = []
+    for chart in chartpress["charts"]:
+        for k, v in chart["images"].items():
+            image_paths.append(v["valuesPath"])
+
+    images = []
+    for image_path in image_paths:
+        image = get_element_from_path(image_path, values)
+        images.append(image["name"] + ":" + image["tag"])
+
+    return images
+
+
+def kind_load_docker_images(kind_cluster, images):
+    """Calls `kind load docker-image <image:tag>` on provided images available locally."""
+
+    for image in images:
+        if not check_output(["docker", "images", "--quiet", image]):
+            continue
+
+        check_output(["kind", "load", "docker-image", "--name", kind_cluster, image])
+        print(f"### Loaded {image}")
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        "--kind-cluster",
+        default="kind",
+        help="Specify a kind cluster to load the docker images into.",
+    )
+    argparser.add_argument(
+        "--values",
+        default="jupyterhub/values.yaml",
+        help="Specify a values.yaml file to look in.",
+    )
+    argparser.add_argument(
+        "--chartpress",
+        default="chartpress.yaml",
+        help="Specify a chartpress.yaml with information about where to look for images.",
+    )
+    args = argparser.parse_args()
+
+    images = extract_images_from_values(
+        chartpress_file=args.chartpress, values_file=args.values
+    )
+    kind_load_docker_images(args.kind_cluster, images)
diff --git a/tools/contributors.py b/tools/contributors.py
--- a/tools/contributors.py
+++ b/tools/contributors.py
@@ -1,30 +1,27 @@
 #!/usr/bin/env python3
 """
-Script to list *all* contributors to a given set of repos,
-between a set of dates. This includes:
+Script to list *all* contributors to a given set of repos, between a set of
+dates. This includes:
 
 - Everyone who has made a commit merged between those dates
 - Everyone who has opened or commented on an issue between those dates
 - Everyone who has opened or commented on a PR between those dates
 
-If you think this misses people who make contributions of a specific
-form, feel free to add them here!
+This script outputs a markdown formatted list of contributors in casefolded name
+(username if name not specified) order.
 
-This script outputs a markdown formatted list of contributors
-in casefolded name (username if name not specified) order.
-
-Since we will be making a ton of requests to the GitHub API, you
-need a GitHub API Token to use this script. The easiest way is to
-just get a Personal Access Token (https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/).
-Treat this token similar to how you would treat a password!
-You can pass this token in with `GITHUB_API_TOKEN` environment
-variable. For example,
+Since we will be making a ton of requests to the GitHub API, you need a GitHub
+API Token to use this script. The easiest way is to just get a Personal Access
+Token
+(https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/).
+Treat this token similar to how you would treat a password! You can pass this
+token in with `GITHUB_API_TOKEN` environment variable. For example,
 
   $  GITHUB_API_TOKEN="your-token" ./tools/contributors.py
 
-Note that if you put a space before your command, it does not
-get stored in your bash history (by default)!. Look up `HISTCONTROL`
-to learn more about this feature of shells.
+Note that if you put a space before your command, it does not get stored in your
+bash history (by default)!. Look up `HISTCONTROL` to learn more about this
+feature of shells.
 
 IMPORTANT:
 You may need to run this script twice or so, utilizing the previous runs cached
@@ -32,10 +29,10 @@
 """
 
 import os
-import dateutil
+from dateutil.parser import parse
+
 import requests_cache
 from github import Github
-from dateutil.parser import parse
 from tqdm import tqdm
 
 requests_cache.install_cache('github')
diff --git a/tools/templates/lint-and-validate.py b/tools/templates/lint-and-validate.py
--- a/tools/templates/lint-and-validate.py
+++ b/tools/templates/lint-and-validate.py
@@ -8,15 +8,14 @@
 
 pip install yamllint
 
-- https://github.com/garethr/kubeval
+- https://github.com/instrumenta/kubeval
 
-LATEST=curl --silent "https://api.github.com/repos/garethr/kubeval/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'
-wget https://github.com/garethr/kubeval/releases/download/$LATEST/kubeval-linux-amd64.tar.gz
+LATEST=curl --silent "https://api.github.com/repos/instrumenta/kubeval/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'
+wget https://github.com/instrumenta/kubeval/releases/download/$LATEST/kubeval-linux-amd64.tar.gz
 tar xf kubeval-darwin-amd64.tar.gz
 mv kubeval /usr/local/bin
 """
 
-
 import os
 import sys
 import argparse
@@ -40,7 +39,7 @@ def check_call(cmd, **kwargs):
         )
         sys.exit(e.returncode)
 
-def lint(yamllint_config, values, kubernetes_version, output_dir, debug):
+def lint(yamllint_config, values, kubernetes_versions, output_dir, debug):
     """Calls `helm lint`, `helm template`, `yamllint` and `kubeval`."""
 
     print("### Clearing output directory")
@@ -52,7 +51,7 @@ def lint(yamllint_config, values, kubernetes_version, output_dir, debug):
     ])
 
     print("### Linting started")
-    print("### 1/4 - helm lint")
+    print("### 1/4 - helm lint: lint helm templates")
     helm_lint_cmd = [
         'helm', 'lint', '../../jupyterhub',
         '--values', values,
@@ -61,7 +60,7 @@ def lint(yamllint_config, values, kubernetes_version, output_dir, debug):
         helm_lint_cmd.append('--debug')
     check_call(helm_lint_cmd)
 
-    print("### 2/4 - helm template")
+    print("### 2/4 - helm template: generate kubernetes resources")
     helm_template_cmd = [
         'helm', 'template', '../../jupyterhub',
         '--values', values,
@@ -71,31 +70,33 @@ def lint(yamllint_config, values, kubernetes_version, output_dir, debug):
         helm_template_cmd.append('--debug')
     check_call(helm_template_cmd)
 
-    print("### 3/4 - yamllint")
+    print("### 3/4 - yamllint: yaml lint generated kubernetes resources")
     check_call([
         'yamllint', '-c', yamllint_config, output_dir
     ])
 
-    print("### 4/4 - kubeval")
-    for filename in glob.iglob(output_dir + '/**/*.yaml', recursive=True):
-        check_call([
-            'kubeval', filename,
-            '--kubernetes-version', kubernetes_version,
-            '--strict',
-        ])
+    print("### 4/4 - kubeval: validate generated kubernetes resources")
+    for kubernetes_version in kubernetes_versions.split(","):
+        print("#### kubernetes_version ", kubernetes_version)
+        for filename in glob.iglob(output_dir + '/**/*.yaml', recursive=True):
+            check_call([
+                'kubeval', filename,
+                '--kubernetes-version', kubernetes_version,
+                '--strict',
+            ])
 
     print()
-    print("### Linting and validation of templates finished: All good!")
+    print("### Linting and validation of helm templates and generated kubernetes resources OK!")
 
 
 if __name__ == '__main__':
     argparser = argparse.ArgumentParser()
     argparser.add_argument('--debug', action='store_true', help='Run helm lint and helm template with the --debug flag')
     argparser.add_argument('--values', default='lint-and-validate-values.yaml', help='Specify Helm values in a YAML file (can specify multiple)')
-    argparser.add_argument('--kubernetes-version', default='1.11.0', help='Version of Kubernetes to validate against')
+    argparser.add_argument('--kubernetes-versions', default='1.15.0', help='List of Kubernetes versions to validate against separated by ","')
     argparser.add_argument('--output-dir', default='rendered-templates', help='Output directory for the rendered templates. Warning: content in this will be wiped.')
     argparser.add_argument('--yamllint-config', default='yamllint-config.yaml', help='Specify a yamllint config')
 
     args = argparser.parse_args()
 
-    lint(args.yamllint_config, args.values, args.kubernetes_version, args.output_dir, args.debug)
+    lint(args.yamllint_config, args.values, args.kubernetes_versions, args.output_dir, args.debug)


Test patch:
diff --git a/ci/test.sh b/ci/test.sh
deleted file mode 100755
--- a/ci/test.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/sh
-
-set -eux
-
-# Is there a standard interface name?
-for iface in eth0 ens4 enp0s3; do
-    IP=$(ifconfig $iface | grep 'inet addr' | cut -d: -f2 | awk '{print $1}');
-    if [ -n "$IP" ]; then
-        echo "IP: $IP"
-        break
-    fi
-done
-if [ -z "$IP" ]; then
-    echo "Failed to get IP, current interfaces:"
-    ifconfig -a
-    exit 2
-fi
-
-TEST_NAMESPACE=jupyterhub-test
-TEST_URL=http://$IP:31212
-
-helm install --name jupyterhub-test --namespace $TEST_NAMESPACE ./jupyterhub/ $Z2JH_HELM_ARGS
-
-echo "waiting for servers to become responsive"
-until curl --fail -s $TEST_URL/hub/api; do
-    kubectl --namespace=$TEST_NAMESPACE describe pod
-    sleep 10
-done
-
-echo "getting jupyterhub version"
-curl -s $TEST_URL/hub/api | grep version
-
-echo "running tests"
-
-display_logs() {
-  echo "***** minikube *****"
-  minikube logs
-  echo "***** node *****"
-  kubectl describe node
-  echo "***** pods *****"
-  kubectl --namespace $TEST_NAMESPACE get pods
-  echo "***** events *****"
-  kubectl --namespace $TEST_NAMESPACE get events
-  echo "***** hub *****"
-  kubectl --namespace $TEST_NAMESPACE logs deploy/hub
-  echo "***** proxy *****"
-  kubectl --namespace $TEST_NAMESPACE logs deploy/proxy
-}
-
-# Run this first to ensure the hub can talk to the proxy
-# (it will automatically retry)
-pytest tests/test_hub_is_ready.py
-
-# Now sleep, and retry again, in case a race condition meant the two were
-# momentarily able to communicate whilst already shutting down
-sleep 1m
-pytest tests/test_hub_is_ready.py
-
-# Hopefully this works now! If tests still failing output logs
-pytest || {
-  r=$?
-  echo "tests failed"
-  display_logs
-  exit $r
-}
-
-# If tests succeeded show all pods to see if any were restarted
-kubectl --namespace $TEST_NAMESPACE get pods
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,3 @@
+# What is this folder about?
+
+This folder contains tests that [pytest](https://docs.pytest.org/en/latest/) will run in our CI/CD pipeline on Travis. These tests must be able to speak directly to a running hub within a Kubernetes cluster etc. In practice, the tests will be set up using a `dev` script.
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,24 +1,30 @@
+"""conftest.py has a special meaning to pytest
+
+ref: https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins
+"""
+
 import os
 import requests
-import pytest
 import uuid
+
+import pytest
 import yaml
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
 def request_data():
     basedir = os.path.dirname(os.path.dirname(__file__))
-    with open(os.path.join(basedir, 'minikube-config.yaml')) as f:
+    with open(os.path.join(basedir, "dev-config.yaml")) as f:
         y = yaml.safe_load(f)
-    token = y['hub']['services']['test']['apiToken']
+    token = y["hub"]["services"]["test"]["apiToken"]
+    host = os.environ.get("Z2JH_PORT_FORWARD_ADDRESS", "localhost")
+    port = os.environ.get("Z2JH_PORT_FORWARD_PORT", "8080")
     return {
-        'token': token,
-        'hub_url': os.getenv('HUB_API_URL', 'http://localhost:31212/hub/api'),
-        'headers': {
-            'Authorization': f'token {token}'
-        },
-        'test_timeout': 300,
-        'request_timeout': 60,
+        "token": token,
+        "hub_url": f'http://{host}:{port}/hub/api',
+        "headers": {"Authorization": f"token {token}"},
+        "test_timeout": 300,
+        "request_timeout": 60,
     }
 
 
@@ -27,39 +33,39 @@ def __init__(self, request_data):
         self.request_data = request_data
 
     def _setup_kwargs(self, kwargs):
-        kwargs['headers'] = kwargs.get('headers', self.request_data['headers'])
-        kwargs['timeout'] = kwargs.get('timeout', self.request_data['request_timeout'])
+        kwargs["headers"] = kwargs.get("headers", self.request_data["headers"])
+        kwargs["timeout"] = kwargs.get("timeout", self.request_data["request_timeout"])
 
     def delete(self, api, **kwargs):
         self._setup_kwargs(kwargs)
-        return requests.delete(self.request_data['hub_url'] + api, **kwargs)
+        return requests.delete(self.request_data["hub_url"] + api, **kwargs)
 
     def get(self, api, **kwargs):
         self._setup_kwargs(kwargs)
-        return requests.get(self.request_data['hub_url'] + api, **kwargs)
+        return requests.get(self.request_data["hub_url"] + api, **kwargs)
 
     def post(self, api, **kwargs):
         self._setup_kwargs(kwargs)
-        return requests.post(self.request_data['hub_url'] + api, **kwargs)
+        return requests.post(self.request_data["hub_url"] + api, **kwargs)
 
     def put(self, api, **kwargs):
         self._setup_kwargs(kwargs)
-        return requests.put(self.request_data['hub_url'] + api, **kwargs)
+        return requests.put(self.request_data["hub_url"] + api, **kwargs)
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope="function")
 def api_request(request_data):
     return JupyterRequest(request_data)
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope="function")
 def jupyter_user(api_request):
     """
     A temporary unique JupyterHub user
     """
-    username = 'testuser-' + str(uuid.uuid4())
-    r = api_request.post('/users/' + username)
+    username = "testuser-" + str(uuid.uuid4())
+    r = api_request.post("/users/" + username)
     assert r.status_code == 201
     yield username
-    r = api_request.delete('/users/' + username)
+    r = api_request.delete("/users/" + username)
     assert r.status_code == 204
diff --git a/tests/test_hub_is_ready.py b/tests/test_hub_is_ready.py
deleted file mode 100644
--- a/tests/test_hub_is_ready.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import requests
-import time
-
-
-def test_hub_can_talk_to_proxy(api_request, request_data):
-    endtime = time.time() + request_data['test_timeout']
-    while time.time() < endtime:
-        try:
-            r = api_request.get('/proxy')
-            if r.status_code == 200:
-                break
-            print(r.json())
-        except requests.RequestException as e:
-            print(e)
-        time.sleep(1)
-    assert r.status_code == 200, 'Failed to get /proxy'
diff --git a/tests/test_spawn.py b/tests/test_spawn.py
--- a/tests/test_spawn.py
+++ b/tests/test_spawn.py
@@ -11,102 +11,200 @@
 
 # load app version of chart
 here = os.path.dirname(os.path.abspath(__file__))
-chart_yaml = os.path.join(here, os.pardir, 'jupyterhub', 'Chart.yaml')
+chart_yaml = os.path.join(here, os.pardir, "jupyterhub", "Chart.yaml")
 
 with open(chart_yaml) as f:
     chart = yaml.safe_load(f)
-    jupyterhub_version = chart['appVersion']
+    jupyterhub_version = chart["appVersion"]
+
 
 def test_api(api_request):
+    """
+    Tests the hub api's root endpoint (/). The hub's version should be returned.
+
+    A typical jupyterhub logging response to this test:
+        
+        [I 2019-09-25 12:03:12.051 JupyterHub log:174] 200 GET /hub/api (test@127.0.0.1) 9.57ms
+    """
+
     print("asking for the hub's version")
-    r = api_request.get('')
+    r = api_request.get("")
     assert r.status_code == 200
     assert r.json().get("version", "version-missing") == jupyterhub_version
 
 
+
 def test_api_info(api_request):
+    """
+    Tests the hub api's /info endpoint. Information about the hub should be
+    returned.
+
+    A typical jupyterhub logging response to this test:
+
+        [I 2019-09-25 12:03:12.086 JupyterHub log:174] 200 GET /hub/api/info (test@127.0.0.1) 10.21ms
+    """
+
     print("asking for the hub information")
-    r = api_request.get('/info')
+    r = api_request.get("/info")
     assert r.status_code == 200
     result = r.json()
-    assert result['spawner']['class'] == 'kubespawner.spawner.KubeSpawner'
+    assert result["spawner"]["class"] == "kubespawner.spawner.KubeSpawner"
+
+
+def test_hub_api_create_user_and_get_information_about_user(api_request, jupyter_user):
+    """
+    Tests the hub api's /users/:user endpoint, both POST and GET.
+
+    A jupyter user is created and commited to the hub database through the
+    jupyter_user pytest fixture declared in conftest.py. Due to this, this first
+    test to use the jupyter_user fixture is actually testing both the pytest
+    fixture that creates the user as well as the ability to get information from
+    the hub about the user. The jupyter_user fixture will automatically clean up
+    the user from the hub's database when this function exits.
+
+    A typical jupyterhub logging response to this test:
+
+        [I 2019-09-25 12:03:12.126 JupyterHub log:174] 201 POST /hub/api/users/testuser-7c70eb90-035b-4d9f-92a5-482e441e307d (test@127.0.0.1) 20.74ms
+        [I 2019-09-25 12:03:12.153 JupyterHub log:174] 200 GET /hub/api/users/testuser-7c70eb90-035b-4d9f-92a5-482e441e307d (test@127.0.0.1) 11.91ms
+        [D 2019-09-25 12:03:12.180 JupyterHub user:240] Creating <class 'kubespawner.spawner.KubeSpawner'> for testuser-7c70eb90-035b-4d9f-92a5-482e441e307d:
+        [I 2019-09-25 12:03:12.204 JupyterHub reflector:199] watching for pods with label selector='component=singleuser-server' in namespace jh-dev
+        [D 2019-09-25 12:03:12.205 JupyterHub reflector:202] Connecting pods watcher
+        [I 2019-09-25 12:03:12.229 JupyterHub reflector:199] watching for events with field selector='involvedObject.kind=Pod' in namespace jh-dev
+        [D 2019-09-25 12:03:12.229 JupyterHub reflector:202] Connecting events watcher
+        [I 2019-09-25 12:03:12.269 JupyterHub log:174] 204 DELETE /hub/api/users/testuser-7c70eb90-035b-4d9f-92a5-482e441e307d (test@127.0.0.1) 98.85ms
+    """
+    print("create a user, and get information about the user")
+    r = api_request.get("/users/" + jupyter_user)
+    assert r.status_code == 200
+    assert r.json()["name"] == jupyter_user
 
 
-def test_api_create_user(api_request, jupyter_user):
-    print("creating the testuser")
-    # Already created by the jupyter_user fixture
-    r = api_request.get('/users/' + jupyter_user)
-    assert r.status_code == 200
-    assert r.json()['name'] == jupyter_user
+def test_hub_api_list_users(api_request, jupyter_user):
+    """
+    Tests the hub api's /users endpoint. Information about users should be
+    returned.
+
+    A typical jupyterhub logging response to this test:
 
+        [I 2019-09-25 12:03:12.303 JupyterHub log:174] 201 POST /hub/api/users/testuser-0d2b0fc9-5ac4-4d8c-8d25-c4545665f81f (test@127.0.0.1) 15.53ms
+        [I 2019-09-25 12:03:12.331 JupyterHub log:174] 200 GET /hub/api/users (test@127.0.0.1) 10.83ms
+        [D 2019-09-25 12:03:12.358 JupyterHub user:240] Creating <class 'kubespawner.spawner.KubeSpawner'> for testuser-0d2b0fc9-5ac4-4d8c-8d25-c4545665f81f:
+        [I 2019-09-25 12:03:12.365 JupyterHub log:174] 204 DELETE /hub/api/users/testuser-0d2b0fc9-5ac4-4d8c-8d25-c4545665f81f (test@127.0.0.1) 18.44ms
+    """
 
-def test_api_list_users(api_request, jupyter_user):
-    print("asking for information")
-    r = api_request.get('/users')
+    print("create a test user, get information about all users, and find the test user")
+    r = api_request.get("/users")
     assert r.status_code == 200
-    assert any(u['name'] == jupyter_user for u in r.json())
+    assert any(u["name"] == jupyter_user for u in r.json())
 
 
-def test_api_request_user_spawn(api_request, jupyter_user, request_data):
-    print("asking kubespawner to spawn testusers singleuser-server pod")
-    r = api_request.post('/users/' + jupyter_user + '/server')
+def test_hub_can_talk_to_proxy(api_request, request_data):
+    """
+    Tests the hub api's /proxy endpoint.
+
+    A typical jupyterhub logging response to this test:
+
+        [I 2019-09-25 12:03:12.395 JupyterHub log:174] 200 GET /hub/api/proxy (test@127.0.0.1) 13.48ms
+    """
+
+    endtime = time.time() + request_data["test_timeout"]
+    while time.time() < endtime:
+        try:
+            r = api_request.get("/proxy")
+            if r.status_code == 200:
+                break
+            print(r.json())
+        except requests.RequestException as e:
+            print(e)
+        time.sleep(1)
+    assert r.status_code == 200, "Failed to get /proxy"
+
+
+def test_hub_api_request_user_spawn(api_request, jupyter_user, request_data):
+    """
+    Tests the hub api's /users/:user/server POST endpoint. A user pod should be
+    created.
+    """
+
+    print("asking kubespawner to spawn a server for a test user")
+    r = api_request.post("/users/" + jupyter_user + "/server")
     assert r.status_code in (201, 202)
     try:
-        server_model = _wait_for_user_to_spawn(api_request, jupyter_user, request_data['test_timeout'])
+        server_model = _wait_for_user_to_spawn(
+            api_request, jupyter_user, request_data["test_timeout"]
+        )
         assert server_model
-        r = requests.get(request_data['hub_url'].partition('/hub/api')[0] + server_model['url'] + "api")
+        r = requests.get(
+            request_data["hub_url"].partition("/hub/api")[0]
+            + server_model["url"]
+            + "api"
+        )
         assert r.status_code == 200
-        assert 'version' in r.json()
+        assert "version" in r.json()
     finally:
-        _delete_server(api_request, jupyter_user, request_data['test_timeout'])
+        _delete_server(api_request, jupyter_user, request_data["test_timeout"])
 
 
-@pytest.mark.skipif(os.getenv('DISABLE_TEST_NETPOL') == '1',
-                    reason="DISABLE_TEST_NETPOL set")
 def test_singleuser_netpol(api_request, jupyter_user, request_data):
-    print("asking kubespawner to spawn a singleuser-server pod to test network policies")
-    r = api_request.post('/users/' + jupyter_user + '/server')
+    """
+    Tests a spawned user pods ability to communicate with allowed and blocked
+    internet locations.
+    """
+
+    print("asking kubespawner to spawn a server for a test user to test network policies")
+    r = api_request.post("/users/" + jupyter_user + "/server")
     assert r.status_code in (201, 202)
     try:
-        server_model = _wait_for_user_to_spawn(api_request, jupyter_user, request_data['test_timeout'])
+        server_model = _wait_for_user_to_spawn(
+            api_request, jupyter_user, request_data["test_timeout"]
+        )
         assert server_model
         print(server_model)
-        pod_name = server_model['state']['pod_name']
+        pod_name = server_model["state"]["pod_name"]
 
-        # Must match CIDR in minikube-netpol.yaml
-        allowed_url = 'http://jupyter.org'
-        blocked_url = 'http://mybinder.org'
+        # Must match CIDR in singleuser.networkPolicy.egress.
+        allowed_url = "http://jupyter.org"
+        blocked_url = "http://mybinder.org"
 
         c = subprocess.run([
-            'kubectl', '--namespace=jupyterhub-test', 'exec', pod_name, '--',
-            'wget', '-q', '-t1', '-T5', allowed_url])
-        assert c.returncode == 0, "Unable to get allowed domain"
+            "kubectl", "exec", pod_name,
+            "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"],
+            "--context", os.environ["Z2JH_KUBE_CONTEXT"],
+            "--",
+            "wget", "--quiet", "--tries=1", "--timeout=3", allowed_url,
+        ])
+        assert c.returncode == 0, "Unable to get allowed domain (or failed to resolve the domain name)"
 
         c = subprocess.run([
-            'kubectl', '--namespace=jupyterhub-test', 'exec', pod_name, '--',
-            'wget', '-q', '-t1', '-T5', blocked_url])
+            "kubectl", "exec", pod_name,
+            "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"],
+            "--context", os.environ["Z2JH_KUBE_CONTEXT"],
+            "--",
+            "wget", "--quiet", "--tries=1", "--timeout=3", blocked_url,
+        ])
         assert c.returncode > 0, "Blocked domain was allowed"
 
     finally:
-        _delete_server(api_request, jupyter_user, request_data['test_timeout'])
+        _delete_server(api_request, jupyter_user, request_data["test_timeout"])
 
 
 def _wait_for_user_to_spawn(api_request, jupyter_user, timeout):
     endtime = time.time() + timeout
     while time.time() < endtime:
-        # FIXME: This can fail with 503! Make it robuster than this!
-        r = api_request.get('/users/' + jupyter_user)
+        # NOTE: If this request fails with a 503 response from the proxy, the
+        #       hub pod has probably crashed by the tests interaction with it.
+        r = api_request.get("/users/" + jupyter_user)
         r.raise_for_status()
         user_model = r.json()
 
         # will be pending while starting,
         # server will be set when ready
-        if '' not in user_model['servers']:
+        if "" not in user_model["servers"]:
             # spawn failed!
             raise RuntimeError("Server never started!")
 
-        server_model = user_model['servers']['']
-        if server_model['ready']:
+        server_model = user_model["servers"][""]
+        if server_model["ready"]:
             return server_model
 
         time.sleep(1)
@@ -114,15 +212,17 @@ def _wait_for_user_to_spawn(api_request, jupyter_user, timeout):
 
 
 def _delete_server(api_request, jupyter_user, timeout):
-    r = api_request.delete('/users/' + jupyter_user + '/server')
+    # NOTE: If this request fails with a 503 response from the proxy, the hub
+    #       pod has probably crashed by the previous tests' interaction with it.
+    r = api_request.delete("/users/" + jupyter_user + "/server")
     assert r.status_code in (202, 204)
 
     endtime = time.time() + timeout
     while time.time() < endtime:
-        r = api_request.get('/users/' + jupyter_user)
+        r = api_request.get("/users/" + jupyter_user)
         r.raise_for_status()
         user_model = r.json()
-        if '' not in user_model['servers']:
+        if "" not in user_model["servers"]:
             return True
         time.sleep(1)
     return False