Repository: jupyterhub/kubespawner

Issue:
Add configuration for annotations on spawned singleuser pods
The spawner supports [configuring extra labels on spawned pods](https://github.com/jupyterhub/kubespawner/blob/06a2e09bc05e873a3e9e8b29d60beceb193fb2b7/kubespawner/spawner.py#L264-L280).

It should also support configuring extra annotations on spawned pods.

Discussion:

Patch:
diff --git a/kubespawner/objects.py b/kubespawner/objects.py
index e2632b7..29ca165 100644
--- a/kubespawner/objects.py
+++ b/kubespawner/objects.py
@@ -36,12 +36,11 @@ def make_pod(
     volumes=[],
     volume_mounts=[],
     labels={},
+    annotations={},
     cpu_limit=None,
     cpu_guarantee=None,
     mem_limit=None,
     mem_guarantee=None,
-    extra_resource_limits=None,
-    extra_resource_guarantees=None,
     lifecycle_hooks=None,
     init_containers=None,
     service_account=None,
@@ -97,6 +96,8 @@ def make_pod(
         String specifying the working directory for the notebook container
       - labels:
         Labels to add to the spawned pod.
+      - annotations:
+        Annotations to add to the spawned pod.
       - cpu_limit:
         Float specifying the max number of CPU cores the user's pod is
         allowed to use.
@@ -131,6 +132,8 @@ def make_pod(
     pod.metadata = V1ObjectMeta()
     pod.metadata.name = name
     pod.metadata.labels = labels.copy()
+    if annotations:
+        pod.metadata.annotations = annotations.copy()
 
     pod.spec = V1PodSpec()
 
@@ -199,19 +202,12 @@ def make_pod(
         notebook_container.resources.requests['cpu'] = cpu_guarantee
     if mem_guarantee:
         notebook_container.resources.requests['memory'] = mem_guarantee
-    if extra_resource_guarantees:
-        for k in extra_resource_guarantees:
-            notebook_container.resources.requests[k] = extra_resource_guarantees[k]
 
     notebook_container.resources.limits = {}
     if cpu_limit:
         notebook_container.resources.limits['cpu'] = cpu_limit
     if mem_limit:
         notebook_container.resources.limits['memory'] = mem_limit
-    if extra_resource_limits:
-        for k in extra_resource_limits:
-            notebook_container.resources.limits[k] = extra_resource_limits[k]
-
     notebook_container.volume_mounts = volume_mounts + hack_volume_mounts
     pod.spec.containers.append(notebook_container)
 
diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py
index c2bffd9..ca13bab 100644
--- a/kubespawner/spawner.py
+++ b/kubespawner/spawner.py
@@ -279,6 +279,23 @@ class KubeSpawner(Spawner):
         """
     )
 
+    singleuser_extra_annotations = Dict(
+        {},
+        config=True,
+        help="""
+        Extra kubernetes annotations to set on the spawned single-user pods.
+
+        The keys and values specified here are added as annotations on the spawned single-user
+        kubernetes pods. The keys and values must both be strings.
+
+        See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ for more
+        info on what annotations are and why you might want to use them!
+
+        {username} and {userid} are expanded to the escaped, dns-label safe
+        username & integer user id respectively, wherever they are used.
+        """
+    )
+
     singleuser_image_spec = Unicode(
         'jupyterhub/singleuser:latest',
         config=True,
@@ -668,28 +685,6 @@ class KubeSpawner(Spawner):
         """
     )
 
-    extra_resource_guarantees = Dict(
-        {},
-        config=True,
-        help="""
-        The dictionary used to request arbitrary resources.
-        Default is None and means no additional resources are requested.
-        For example, to request 3 Nvidia GPUs
-            `{"nvidia.com/gpu": "3"}`
-        """
-    )
-
-    extra_resource_limits = Dict(
-        {},
-        config=True,
-        help="""
-        The dictionary used to limit arbitrary resources.
-        Default is None and means no additional resources are limited.
-        For example, to add a limit of 3 Nvidia GPUs
-            `{"nvidia.com/gpu": "3"}`
-        """
-    )
-
     def _expand_user_properties(self, template):
         # Make sure username and servername match the restrictions for DNS labels
         safe_chars = set(string.ascii_lowercase + string.digits)
@@ -764,6 +759,7 @@ class KubeSpawner(Spawner):
             real_cmd = None
 
         labels = self._build_pod_labels(self._expand_all(self.singleuser_extra_labels))
+        annotations = self._expand_all(self.singleuser_extra_annotations)
 
         return make_pod(
             name=self.pod_name,
@@ -781,12 +777,11 @@ class KubeSpawner(Spawner):
             volume_mounts=self._expand_all(self.volume_mounts),
             working_dir=self.singleuser_working_dir,
             labels=labels,
+            annotations=annotations,
             cpu_limit=self.cpu_limit,
             cpu_guarantee=self.cpu_guarantee,
             mem_limit=self.mem_limit,
             mem_guarantee=self.mem_guarantee,
-            extra_resource_limits=self.extra_resource_limits,
-            extra_resource_guarantees=self.extra_resource_guarantees,
             lifecycle_hooks=self.singleuser_lifecycle_hooks,
             init_containers=self.singleuser_init_containers,
             service_account=self.singleuser_service_account,


Test patch:
diff --git a/tests/test_objects.py b/tests/test_objects.py
index cc0bde8..5369fb0 100644
--- a/tests/test_objects.py
+++ b/tests/test_objects.py
@@ -90,6 +90,49 @@ def test_make_labeled_pod():
         "apiVersion": "v1"
     }
 
+def test_make_annotated_pod():
+    """
+    Test specification of the simplest possible pod specification with annotations
+    """
+    assert api_client.sanitize_for_serialization(make_pod(
+        name='test',
+        image_spec='jupyter/singleuser:latest',
+        cmd=['jupyterhub-singleuser'],
+        port=8888,
+        image_pull_policy='IfNotPresent',
+        annotations={"test": "true"}
+    )) == {
+        "metadata": {
+            "name": "test",
+            "annotations": {"test": "true"},
+            "labels": {},
+        },
+        "spec": {
+            "securityContext": {},
+            "containers": [
+                {
+                    "env": [],
+                    "name": "notebook",
+                    "image": "jupyter/singleuser:latest",
+                    "imagePullPolicy": "IfNotPresent",
+                    "args": ["jupyterhub-singleuser"],
+                    "ports": [{
+                        "name": "notebook-port",
+                        "containerPort": 8888
+                    }],
+                    'volumeMounts': [{'name': 'no-api-access-please', 'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'readOnly': True}],
+                    "resources": {
+                        "limits": {},
+                        "requests": {}
+                    }
+                }
+            ],
+            'volumes': [{'name': 'no-api-access-please', 'emptyDir': {}}],
+        },
+        "kind": "Pod",
+        "apiVersion": "v1"
+    }
+
 def test_make_pod_with_image_pull_secrets():
     """
     Test specification of the simplest possible pod specification
@@ -631,65 +674,6 @@ def test_make_pod_with_extra_containers():
         "apiVersion": "v1"
     }
 
-def test_make_pod_with_extra_resources():
-    """
-    Test specification of extra resources (like GPUs)
-    """
-    assert api_client.sanitize_for_serialization(make_pod(
-        name='test',
-        image_spec='jupyter/singleuser:latest',
-        cpu_limit=2,
-        cpu_guarantee=1,
-        extra_resource_limits={"nvidia.com/gpu": "5", "k8s.io/new-resource": "1"},
-        extra_resource_guarantees={"nvidia.com/gpu": "3"},
-        cmd=['jupyterhub-singleuser'],
-        port=8888,
-        mem_limit='1Gi',
-        mem_guarantee='512Mi',
-        image_pull_policy='IfNotPresent',
-        image_pull_secret="myregistrykey",
-        node_selector={"disk": "ssd"}
-    )) == {
-        "metadata": {
-            "name": "test",
-            "labels": {},
-        },
-        "spec": {
-            "securityContext": {},
-            "imagePullSecrets": [{"name": "myregistrykey"}],
-            "nodeSelector": {"disk": "ssd"},
-            "containers": [
-                {
-                    "env": [],
-                    "name": "notebook",
-                    "image": "jupyter/singleuser:latest",
-                    "imagePullPolicy": "IfNotPresent",
-                    "args": ["jupyterhub-singleuser"],
-                    "ports": [{
-                        "name": "notebook-port",
-                        "containerPort": 8888
-                    }],
-                    'volumeMounts': [{'name': 'no-api-access-please', 'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'readOnly': True}],
-                    "resources": {
-                        "limits": {
-                            "cpu": 2,
-                            "memory": '1Gi',
-                            "nvidia.com/gpu": "5",
-                            "k8s.io/new-resource": "1"
-                        },
-                        "requests": {
-                            "cpu": 1,
-                            "memory": '512Mi',
-                            "nvidia.com/gpu": "3"
-                        }
-                    }
-                }
-            ],
-            'volumes': [{'name': 'no-api-access-please', 'emptyDir': {}}],
-        },
-        "kind": "Pod",
-        "apiVersion": "v1"
-    }
 
 def test_make_pvc_simple():
     """