Repository: dask/distributed

Issue:
Fine performance metrics: Store data on the Scheduler
- Part of #7665

There are a two main options with the data stored in `Worker.digests_total`: either we keep it there and publish it to Prometheus from the workers, or we aggregate it on the scheduler first and move from there.

Pushing the data to the scheduler opens to displaying the data on Bokeh and Jupyter, instead of just Prometheus. It also gets rid of unwanted per-worker cardinality, which would be very overwhelming for Prometheus to store.
Finally, it opens to enriching the data with scheduler-only information (e.g. #7672).

#### Low level design
Add an extra defaultdict, `Worker.digests_total_new`, which is emptied and sent to the scheduler at every heartbeat.
This will be collected on the scheduler in `Scheduler.cumulative_worker_metrics`, without the worker information.

This means that:
- At every heartbeat, workers will send a relatively small dict that is just about recently finished tasks
- if you lose a worker, you lose only the data since the latest heartbeat
- it works just fine with adaptive clusters
- you can trivially capture end-to-end metrics of a run from the client (#7667)



Discussion:

Resolving patch:
diff --git a/distributed/core.py b/distributed/core.py
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -384,6 +384,7 @@ def stop() -> bool:
         # In case crick is not installed, also log cumulative totals (reset at server
         # restart) and local maximums (reset by prometheus poll)
         self.digests_total = defaultdict(float)
+        self.digests_total_since_heartbeat = defaultdict(float)
         self.digests_max = defaultdict(float)
 
         self.counters = defaultdict(Counter)
@@ -960,6 +961,8 @@ def digest_metric(self, name: Hashable, value: float) -> None:
             self.digests[name].add(value)
         # Cumulative data (reset by server restart)
         self.digests_total[name] += value
+        # Cumulative data sent to scheduler and reset on heartbeat
+        self.digests_total_since_heartbeat[name] += value
         # Local maximums (reset by Prometheus poll)
         self.digests_max[name] = max(self.digests_max[name], value)
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3442,6 +3442,8 @@ def __init__(
         self.bandwidth_workers = defaultdict(float)
         self.bandwidth_types = defaultdict(float)
 
+        self.cumulative_worker_metrics = defaultdict(float)
+
         if not preload:
             preload = dask.config.get("distributed.scheduler.preload")
         if not preload_argv:
@@ -4045,6 +4047,9 @@ def heartbeat_worker(
                     ws.executing[ts] = duration
                     ts.prefix.add_exec_time(duration)
 
+        for name, value in metrics["digests_total_since_heartbeat"].items():
+            self.cumulative_worker_metrics[name] += value
+
         ws.metrics = metrics
 
         # Calculate RSS - dask keys, separating "old" and "new" usage
diff --git a/distributed/worker.py b/distributed/worker.py
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1042,6 +1042,7 @@ async def get_metrics(self) -> dict:
                 "workers": dict(self.bandwidth_workers),
                 "types": keymap(typename, self.bandwidth_types),
             },
+            digests_total_since_heartbeat=dict(self.digests_total_since_heartbeat),
             managed_bytes=self.state.nbytes,
             spilled_bytes={
                 "memory": spilled_memory,
@@ -1058,6 +1059,8 @@ async def get_metrics(self) -> dict:
             event_loop_interval=self._tick_interval_observed,
         )
 
+        self.digests_total_since_heartbeat.clear()
+
         monitor_recent = self.monitor.recent()
         # Convert {foo.bar: 123} to {foo: {bar: 123}}
         for k, v in monitor_recent.items():


Test patch:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2344,6 +2344,24 @@ async def test_idle_timeout_no_workers(c, s):
     assert s.check_idle()
 
 
+@gen_cluster(client=True)
+async def test_cumulative_worker_metrics(c, s, a, b):
+    # Race condition: metrics that are updated while idle may or may not be there already
+    assert s.cumulative_worker_metrics.keys() in (set(), {"latency"})
+    await c.submit(inc, 1, key="do_work")
+
+    await a.heartbeat()
+    await b.heartbeat()
+
+    metrics = s.cumulative_worker_metrics
+
+    # Subset of expected keys
+    assert "latency" in metrics
+    assert ("execute", "do_work", "deserialize", "seconds") in metrics
+
+    assert all(isinstance(value, float) for value in metrics.values())
+
+
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})
 async def test_bandwidth(c, s, a, b):
     start = s.bandwidth