Repository: jupyterhub/jupyterhub

Issue:
Add server_poll_duration_seconds prometheus metric
We use [prometheus](https://github.com/prometheus/client_python) to expose metrics about JupyterHub operations. Currently we expose HTTP request stats and time it takes to spawn servers. We should expose more metrics!

One useful metric is 'time taken for JupyterHub to poll if server is running'. Periodically (and at startup), JupyterHub goes through the list of servers it thinks are running & verifies that they *are* running. It does so by polling them through the spawner. It would be great to record how much time this takes.

 It should be a [histogram](https://github.com/prometheus/client_python#histogram) metric. We should observe the timing whenever a server is stopped. This is similar to the SERVER_SPAWN_DURATION_SECONDS metric, which is a histogram that observes the amount of time servers take to start.

- [ ] Declare the metric in metrics.py
- [ ] Observe time it takes for a spawner poll to happen


Discussion:

Resolving patch:
diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py
--- a/jupyterhub/handlers/base.py
+++ b/jupyterhub/handlers/base.py
@@ -32,6 +32,7 @@
 from ..metrics import (
     SERVER_SPAWN_DURATION_SECONDS, ServerSpawnStatus,
     PROXY_ADD_DURATION_SECONDS, ProxyAddStatus,
+    SERVER_POLL_DURATION_SECONDS, ServerPollStatus,
     RUNNING_SERVERS
 )
 
@@ -821,13 +822,19 @@ def abort():
 
             # start has finished, but the server hasn't come up
             # check if the server died while we were waiting
+            poll_start_time = time.perf_counter()
             status = await spawner.poll()
+            SERVER_POLL_DURATION_SECONDS.labels(
+                status=ServerPollStatus.from_status(status)
+            ).observe(time.perf_counter() - poll_start_time)
+
             if status is not None:
                 toc = IOLoop.current().time()
                 self.statsd.timing('spawner.failure', (toc - tic) * 1000)
                 SERVER_SPAWN_DURATION_SECONDS.labels(
                     status=ServerSpawnStatus.failure
                 ).observe(time.perf_counter() - spawn_start_time)
+
                 raise web.HTTPError(500, "Spawner failed to start [status=%s]. The logs for %s may contain details." % (
                     status, spawner._log_name))
 
@@ -848,9 +855,17 @@ def abort():
     async def user_stopped(self, user, server_name):
         """Callback that fires when the spawner has stopped"""
         spawner = user.spawners[server_name]
+
+        poll_start_time = time.perf_counter()
         status = await spawner.poll()
+        SERVER_POLL_DURATION_SECONDS.labels(
+            status=ServerPollStatus.from_status(status)
+        ).observe(time.perf_counter() - poll_start_time)
+
+
         if status is None:
             status = 'unknown'
+
         self.log.warning("User %s server stopped, with exit code: %s",
             user.name, status,
         )
@@ -1152,10 +1167,13 @@ async def get(self, user_name, user_path):
 
             # spawn has supposedly finished, check on the status
             if spawner.ready:
+                poll_start_time = time.perf_counter()
                 status = await spawner.poll()
+                SERVER_POLL_DURATION_SECONDS.labels(
+                    status=ServerPollStatus.from_status(status)
+                ).observe(time.perf_counter() - poll_start_time)
             else:
                 status = 0
-
             # server is not running, trigger spawn
             if status is not None:
                 if spawner.options_form:
diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py
--- a/jupyterhub/metrics.py
+++ b/jupyterhub/metrics.py
@@ -37,7 +37,7 @@
 
 RUNNING_SERVERS = Gauge(
     'running_servers',
-    'the number of user servers currently running',
+    'the number of user servers currently running'
 )
 
 RUNNING_SERVERS.set(0)
@@ -79,6 +79,34 @@ def __str__(self):
 for s in ProxyAddStatus:
     PROXY_ADD_DURATION_SECONDS.labels(status=s)
 
+
+SERVER_POLL_DURATION_SECONDS = Histogram(
+    'server_poll_duration_seconds',
+    'time taken to poll if server is running',
+    ['status']
+)
+
+class ServerPollStatus(Enum):
+    """
+    Possible values for 'status' label of SERVER_POLL_DURATION_SECONDS
+    """
+    running = 'running'
+    stopped = 'stopped'
+
+    @classmethod
+    def from_status(cls, status):
+        """Return enum string for a given poll status"""
+        if status is None:
+            return cls.running
+        return cls.stopped
+
+    def __str__(self):
+        return self.value
+
+for s in ServerPollStatus:
+    SERVER_POLL_DURATION_SECONDS.labels(status=s)
+
+
 def prometheus_log_method(handler):
     """
     Tornado log handler for recording RED metrics.


Test patch: