Repository: dask/distributed

Issue:
Create Prometheus Endpoint
We might consider publishing metrics at an endpoint for [Prometheus](https://prometheus.io).  This would allow standard logging and visualization tools to easily get some information about Dask.

cc @jacobtomlinson 

It might be helpful to look at [our code to host normal static pages](https://github.com/dask/distributed/blob/master/distributed/bokeh/scheduler_html.py)


Discussion:
In order to implement this all the metrics we wish to cover need to be exposed in the [prometheus text based format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format) at `/metrics`.

It looks like the [`CountsJSON`](https://github.com/dask/distributed/blob/master/distributed/bokeh/scheduler_html.py#L96) endpoint is already providing the metrics we would want, so a new endpoint needs creating with the same data in the right format.

There is a prometheus [python client library](https://github.com/prometheus/client_python) which can handle the text formatting.
The `CountsJSON` endpoint is very interesting on the scheduler side of things. Is there also a similar thing for the workers? Are there any metrics besides CPU/RAM usage that we want to scrape from the workers? I could think of currently:

- Stored bytes (RAM & disk)
- running tasks
There isn't anything like CountsJSON for workers, but there could be.  We
could do this by replicating the content in
distributed/bokeh/scheduler_html.py file, and then adding a listen method
to the disributed/bokeh/worker.py::BokehWorker class that is similar to the
listen method on the distributed/bokeh/scheduler.py::BokehScheduler class.

On Thu, Nov 22, 2018 at 9:56 AM Uwe L. Korn <notifications@github.com>
wrote:

> The CountsJSON endpoint is very interesting on the scheduler side of
> things. Is there also a similar thing for the workers? Are there any
> metrics besides CPU/RAM usage that we want to scrape from the workers? I
> could think of currently:
>
>    - Stored bytes (RAM & disk)
>    - running tasks
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/2332#issuecomment-441054479>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszKHaD5Fr-llGm7T-fOMHunG8RQwAks5uxrsygaJpZM4YHVQK>
> .
>

We'll probably need to add a normal HTTP server to the workers for this.  The commit that did this for the scheduler is here: 74cebfb9eeaef0b51963879403d094275e3b2b94

Resolving patch:
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -163,6 +163,26 @@ def get(self):
         self.write(result)
 
 
+class PrometheusHandler(RequestHandler):
+    def __init__(self, *args, **kwargs):
+        import prometheus_client # keep out of global namespace
+        self.prometheus_client = prometheus_client
+
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
+        self.workers = self.prometheus_client.Gauge('workers_total',
+            'Total number of workers.',
+            namespace='scheduler')
+        self.clients = self.prometheus_client.Gauge('clients_total',
+            'Total number of clients.',
+            namespace='scheduler')
+
+    def get(self):
+        self.workers.set(len(self.server.workers))
+        self.clients.set(len(self.server.clients))
+
+        self.write(self.prometheus_client.generate_latest())
+
+
 routes = [
         (r'info/main/workers.html', Workers),
         (r'info/worker/(.*).html', Worker),
@@ -175,6 +195,7 @@ def get(self):
         (r'json/identity.json', IdentityJSON),
         (r'json/index.html', IndexJSON),
         (r'individual-plots.json', IndividualPlots),
+        (r'metrics', PrometheusHandler),
 ]
 
 
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -678,6 +678,19 @@ def __init__(self, worker, io_loop=None, prefix='', **kwargs):
         self.loop = io_loop or worker.loop
         self.server = None
 
+    @property
+    def extra(self):
+        return merge({'prefix': self.prefix}, template_variables)
+
     @property
     def my_server(self):
         return self.worker
+
+    def listen(self, *args, **kwargs):
+        super(BokehWorker, self).listen(*args, **kwargs)
+
+        from .worker_html import routes
+        handlers = [(self.prefix + '/' + url, cls, {'server': self.my_server, 'extra': self.extra})
+                    for url, cls in routes]
+
+        self.server._tornado.add_handlers(r'.*', handlers)
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
new file mode 100644
--- /dev/null
+++ b/distributed/bokeh/worker_html.py
@@ -0,0 +1,41 @@
+import os
+
+from tornado import web
+
+dirname = os.path.dirname(__file__)
+
+
+class RequestHandler(web.RequestHandler):
+    def initialize(self, server=None, extra=None):
+        self.server = server
+        self.extra = extra or {}
+
+    def get_template_path(self):
+        return os.path.join(dirname, 'templates')
+
+
+class PrometheusHandler(RequestHandler):
+    def __init__(self, *args, **kwargs):
+        import prometheus_client # keep out of global namespace
+        self.prometheus_client = prometheus_client
+
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
+        # Add metrics like this:
+        # self.workers = self.prometheus_client.Gauge('memory_bytes',
+        #    'Total memory.',
+        #    namespace='worker')
+
+    def get(self):
+        # Example metric update
+        # self.workers.set(0.)
+
+        self.write(self.prometheus_client.generate_latest())
+
+
+routes = [
+        (r'metrics', PrometheusHandler),
+]
+
+
+def get_handlers(server):
+    return [(url, cls, {'server': server}) for url, cls in routes]


Test patch:
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -60,3 +60,13 @@ def test_prefix(c, s, a, b):
             json.loads(body)
         else:
             assert xml.etree.ElementTree.fromstring(body) is not None
+
+
+@gen_cluster(client=True,
+             check_new_threads=False,
+             scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+def test_prometheus(c, s, a, b):
+    http_client = AsyncHTTPClient()
+    response = yield http_client.fetch('http://localhost:%d/metrics'
+                                       % s.services['bokeh'].port)
+    assert response.code == 200
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -118,3 +118,16 @@ def test_CommunicatingStream(c, s, a, b):
             len(first(bb.outgoing.data.values())))
     assert (len(first(aa.incoming.data.values())) and
             len(first(bb.incoming.data.values())))
+
+
+@gen_cluster(client=True,
+             check_new_threads=False,
+             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+def test_prometheus(c, s, a, b):
+    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
+
+    http_client = AsyncHTTPClient()
+    for suffix in ['metrics']:
+        response = yield http_client.fetch('http://localhost:%d/%s'
+                                           % (a.services['bokeh'].port, suffix))
+        assert response.code == 200