Repository: dask/distributed

Issue:
Fine performance metrics dashboard crashes w/ spill activity
This code is sized so that it triggers spill/unspill activity:
```python
import dask.array as da
import distributed

client = distributed.Client(n_workers=4, threads_per_worker=1, memory_limit="2 GiB")
a = da.random.random((14_000, 14_000))
b = (a @ a.T).sum()
b.compute()
```
If I open the Fine Performance Metrics dashboard after I run it, I get a blank page with 500 Internal server error, while on the stderr I get
```
BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('activity', 11), ('angle', 11), ('color', 0), ('text', 11), ('value', 11)
2023-06-01 15:33:53,229 - bokeh.core.property.validation - ERROR - Keyword argument sequences for broadcasting must all be the same lengths. Got lengths: [0, 11]
Traceback (most recent call last):
  File "/home/crusaderky/github/distributed/distributed/utils.py", line 760, in wrapper
    return func(*args, **kwargs)
  File "/home/crusaderky/github/distributed/distributed/dashboard/components/scheduler.py", line 3510, in update
    task_exec_barchart = self._build_task_execution_by_prefix_chart(
  File "/home/crusaderky/github/distributed/distributed/dashboard/components/scheduler.py", line 3598, in _build_task_execution_by_prefix_chart
    renderers = barchart.vbar_stack(
  File "/home/crusaderky/miniconda3/envs/distributed39/lib/python3.9/site-packages/bokeh/plotting/figure.py", line 588, in vbar_stack
    for kw in double_stack(stackers, "bottom", "top", **kw):
  File "/home/crusaderky/miniconda3/envs/distributed39/lib/python3.9/site-packages/bokeh/plotting/_stack.py", line 83, in double_stack
    raise ValueError("Keyword argument sequences for broadcasting must all be the same lengths. Got lengths: %r" % sorted(list(lengths)))
ValueError: Keyword argument sequences for broadcasting must all be the same lengths. Got lengths: [0, 11]
```

I reproduced the above both with bokeh 2 and bokeh 3.

The problem disappears if I size either my problem or the cluster so that spill/unspill events are never generated. 
I'm attaching a trimmed-down dump of the metrics, which will allow you to reproduce it deterministically. Just copy-paste the below in a jupyter notebook and open the fine performance metrics dashboard page:
```python
import distributed

client = distributed.Client()
metrics = {
    ("execute", "chunk_sum-aggregate-sum", "disk-read", "seconds"): 1.0051379720016484,
    ("execute", "chunk_sum-aggregate-sum", "disk-read", "count"): 16.0,
    ("execute", "chunk_sum-aggregate-sum", "disk-read", "bytes"): 2059931767.0,
    ("execute", "chunk_sum-aggregate-sum", "disk-write", "seconds"): 0.1692888050001784,
    ("execute", "chunk_sum-aggregate-sum", "disk-write", "count"): 2.0,
    ("execute", "chunk_sum-aggregate-sum", "disk-write", "bytes"): 268435938.0,
}
client.cluster.scheduler.cumulative_worker_metrics.clear()
client.cluster.scheduler.cumulative_worker_metrics.update(metrics)
```



Discussion:

Resolving patch:
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -46,7 +46,7 @@
 )
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.models.widgets.markups import Div
-from bokeh.palettes import Viridis11, small_palettes
+from bokeh.palettes import Viridis11, YlGnBu9, interp_palette
 from bokeh.plotting import figure
 from bokeh.themes import Theme
 from bokeh.transform import cumsum, factor_cmap, linear_cmap, stack
@@ -3522,7 +3522,7 @@ def update(self):
 
     def _build_task_execution_by_activity_chart(
         self, task_exec_data: defaultdict[str, list]
-    ) -> figure:
+    ) -> None:
         piechart_data = {}
         piechart_data["value"] = [
             sum(task_exec_data[f"{op}_{self.unit_selected}"])
@@ -3542,9 +3542,7 @@ def _build_task_execution_by_activity_chart(
             * math.pi
             for activity in self.task_activities
         ]
-        piechart_data["color"] = small_palettes["YlGnBu"].get(
-            len(self.task_activities), []
-        )
+        piechart_data["color"] = self._get_palette(len(self.task_activities))
         piechart_data["activity"] = self.task_activities
         self.task_exec_by_activity_src.data = piechart_data
 
@@ -3576,29 +3574,46 @@ def _build_task_execution_by_activity_chart(
 
     def _build_task_execution_by_prefix_chart(
         self, task_exec_data: defaultdict[str, list]
-    ) -> figure:
-        barchart = figure(
-            x_range=task_exec_data["functions"],
-            height=500,
-            sizing_mode="scale_both",
-            title="Task execution, by function",
-            tools="pan,wheel_zoom,box_zoom,reset",
-        )
-        barchart.yaxis.visible = False
-        barchart.xaxis.major_label_orientation = 0.2
-        barchart.grid.grid_line_color = None
+    ) -> None:
+        functions = task_exec_data["functions"]
+        base_tools = "pan,wheel_zoom,box_zoom,reset"
+
+        if self.task_exec_by_prefix_chart is None:
+            barchart = figure(
+                x_range=functions,
+                height=500,
+                sizing_mode="scale_both",
+                title="Task execution, by function",
+                tools=base_tools,
+            )
+            barchart.yaxis.visible = False
+            barchart.xaxis.major_label_orientation = 0.2
+            barchart.grid.grid_line_color = None
+            self.task_exec_by_prefix_chart = barchart
+
+        if self.task_exec_by_prefix_chart.x_range.factors != functions:
+            self.substantial_change = True
+            self.task_exec_by_prefix_chart.x_range = FactorRange(*functions)
+
         stackers = [
             name for name in task_exec_data if name.endswith(self.unit_selected)
         ]
-        if stackers:
-            renderers = barchart.vbar_stack(
+
+        if stackers and stackers != getattr(self, "_prev_stackers", []):
+            self._prev_stackers = stackers
+            renderers = self.task_exec_by_prefix_chart.vbar_stack(
                 stackers,
                 x="functions",
                 width=0.9,
                 source=self.task_exec_by_prefix_src,
-                color=small_palettes["YlGnBu"].get(len(self.task_activities), []),
+                color=self._get_palette(len(self.task_activities)),
                 legend_label=self.task_activities,
             )
+
+            # Create hovertools ontop of base tools
+            tools = self.task_exec_by_prefix_chart.tools
+            self.task_exec_by_prefix_chart.tools = tools[: len(base_tools.split(","))]
+
             for vbar in renderers:
                 tooltips = [
                     (
@@ -3607,19 +3622,15 @@ def _build_task_execution_by_prefix_chart(
                     ),
                     ("function", "@functions"),
                 ]
-                barchart.add_tools(HoverTool(tooltips=tooltips, renderers=[vbar]))
-
-            if any(
-                len(self.task_exec_by_prefix_src.data[k]) != len(task_exec_data[k])
-                for k in self.task_exec_by_prefix_src.data
-            ):
-                self.substantial_change = True
+                self.task_exec_by_prefix_chart.add_tools(
+                    HoverTool(tooltips=tooltips, renderers=[vbar])
+                )
 
-            self.task_exec_by_prefix_src.data = dict(task_exec_data)
-            barchart.renderers = renderers
-        self.task_exec_by_prefix_chart = barchart
+            self.substantial_change = True
+            self.task_exec_by_prefix_chart.renderers = renderers
+        self.task_exec_by_prefix_src.data = dict(task_exec_data)
 
-    def _build_senddata_chart(self, senddata: defaultdict[str, list]) -> figure:
+    def _build_senddata_chart(self, senddata: defaultdict[str, list]) -> None:
         piedata = {}
         piedata["activity"] = senddata["activity"]
         piedata["value"] = [
@@ -3636,7 +3647,7 @@ def _build_senddata_chart(self, senddata: defaultdict[str, list]) -> figure:
             * math.pi
             for op in piedata["activity"]
         ]
-        piedata["color"] = small_palettes["YlGnBu"].get(len(piedata["activity"]), [])
+        piedata["color"] = self._get_palette(len(piedata["activity"]))
 
         self.sendsrc.data = piedata
 
@@ -3665,6 +3676,9 @@ def _build_senddata_chart(self, senddata: defaultdict[str, list]) -> figure:
             senddata_piechart.grid.grid_line_color = None
             self.senddata_by_activity_chart = senddata_piechart
 
+    def _get_palette(self, n: int) -> list[str]:
+        return interp_palette(YlGnBu9, n)
+
 
 class Contention(DashboardComponent):
     """


Test patch:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -344,6 +344,25 @@ async def test_FinePerformanceMetrics(c, s, a, b):
     assert cl.task_exec_data["functions"] == ["slowinc"]
 
 
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
+async def test_FinePerformanceMetrics_simulated_spill_no_crash(c, s, a, b):
+    metrics = {
+        ("execute", "inc", "disk-read", "seconds"): 1.0,
+        ("execute", "inc", "disk-read", "count"): 16.0,
+        ("execute", "inc", "disk-read", "bytes"): 2059931767.0,
+        ("execute", "inc", "disk-write", "seconds"): 0.12,
+        ("execute", "inc", "disk-write", "count"): 2.0,
+        ("execute", "inc", "disk-write", "bytes"): 268435938.0,
+    }
+    s.cumulative_worker_metrics.clear()
+    s.cumulative_worker_metrics.update(metrics)
+    http_client = AsyncHTTPClient()
+    response = await http_client.fetch(
+        f"http://localhost:{s.http_server.port}/individual-fine-performance-metrics"
+    )
+    assert response.code == 200
+
+
 @gen_cluster(client=True)
 async def test_ClusterMemory(c, s, a, b):
     cl = ClusterMemory(s)