Repository: dask/distributed

Issue:
Compute time per task with too many task types
I'm loving the new compute time per task plot.  I think that we could improve it in two ways:

1.  Format times better.  Often I see times like `1.528528e4 s`.  We might do this on the server side with `dask.utils.format_time`, or on the Bokeh side, however they manage formatting. 
2.  We should probably only show the top few entries:

![image](https://user-images.githubusercontent.com/306380/85952538-cdbb8f00-b91e-11ea-8fcd-4eba18d40c1e.png)

Actually, this may be one situation where a pie chart makes more sense.  We don't need to give the same horizontal space to every task type.


Discussion:
@quasiben @jsignell 
I think we could do away with the legend entirely if it's just duplicating the x-ticks. I am personally fine with a pie chart for this, but people hate them so much, I'm not sure it's worth it.
Heh, yeah.  This seems like maybe one of those situations where a pie chart
might actually make sense though.  The amount of visual emphasis matches
the question we're trying to figure out, and the distribution of values
that we tend to see (there are often only one or two relevant components to
cost, and pie charts do well in that range).

On Mon, Jun 29, 2020 at 6:18 AM Julia Signell <notifications@github.com>
wrote:

> I think we could do away with the legend entirely if it's just duplicating
> the x-ticks. I am personally fine with a pie chart for this, but people
> hate them so much, I'm not sure it's worth it.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/3930#issuecomment-651114991>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTDOMOY7KUJTRM53LDTRZCICTANCNFSM4OKTCW4Q>
> .
>

Not pushing hard for pie charts though, we could also just truncate
anything less than 2% or something.

On Mon, Jun 29, 2020 at 6:50 AM Matthew Rocklin <mrocklin@gmail.com> wrote:

> Heh, yeah.  This seems like maybe one of those situations where a pie
> chart might actually make sense though.  The amount of visual emphasis
> matches the question we're trying to figure out, and the distribution of
> values that we tend to see (there are often only one or two relevant
> components to cost, and pie charts do well in that range).
>
> On Mon, Jun 29, 2020 at 6:18 AM Julia Signell <notifications@github.com>
> wrote:
>
>> I think we could do away with the legend entirely if it's just
>> duplicating the x-ticks. I am personally fine with a pie chart for this,
>> but people hate them so much, I'm not sure it's worth it.
>>
>> —
>> You are receiving this because you authored the thread.
>> Reply to this email directly, view it on GitHub
>> <https://github.com/dask/distributed/issues/3930#issuecomment-651114991>,
>> or unsubscribe
>> <https://github.com/notifications/unsubscribe-auth/AACKZTDOMOY7KUJTRM53LDTRZCICTANCNFSM4OKTCW4Q>
>> .
>>
>


Resolving patch:
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -26,12 +26,15 @@
     BoxSelectTool,
     GroupFilter,
     CDSView,
+    Tabs,
+    Panel,
+    Title,
 )
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
 from bokeh.themes import Theme
-from bokeh.transform import factor_cmap, linear_cmap
+from bokeh.transform import factor_cmap, linear_cmap, cumsum
 from bokeh.io import curdoc
 import dask
 from dask import config
@@ -440,7 +443,7 @@ def name(address):
             update(self.source, result)
 
 
-class ComputerPerKey(DashboardComponent):
+class ComputePerKey(DashboardComponent):
     """ Bar chart showing time spend in action by key prefix"""
 
     def __init__(self, scheduler, **kwargs):
@@ -456,6 +459,8 @@ def __init__(self, scheduler, **kwargs):
 
             compute_data = {
                 "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
+                "angles": [3.14, 0.785],
                 "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
                 "names": ["sum", "sum_partial"],
             }
@@ -477,14 +482,13 @@ def __init__(self, scheduler, **kwargs):
                 top="times",
                 width=0.7,
                 color="color",
-                legend_field="names",
             )
 
             fig.y_range.start = 0
             fig.min_border_right = 20
             fig.min_border_bottom = 60
             fig.yaxis.axis_label = "Time (s)"
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
             rect.nonselection_glyph = None
@@ -499,13 +503,76 @@ def __init__(self, scheduler, **kwargs):
             hover.tooltips = """
             <div>
                 <p><b>Name:</b> @names</p>
-                <p><b>Time:</b> @times s</p>
+                <p><b>Time:</b> @formatted_time</p>
             </div>
             """
             hover.point_policy = "follow_mouse"
             fig.add_tools(hover)
 
+            fig.add_layout(
+                Title(
+                    text="Note: tasks less than 2% of max are not displayed",
+                    text_font_style="italic",
+                ),
+                "below",
+            )
+
             self.fig = fig
+            tab1 = Panel(child=fig, title="Bar Chart")
+
+            compute_wedge_data = {
+                "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
+                "angles": [1.4, 0.8],
+                "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
+                "names": ["sum", "sum_partial"],
+            }
+
+            fig2 = figure(
+                title="Compute Time Per Task",
+                tools="",
+                id="bk-Compute-by-key-pie",
+                name="compute_time_per_key-pie",
+                x_range=(-0.5, 1.0),
+                **kwargs,
+            )
+
+            wedge = fig2.wedge(
+                x=0,
+                y=1,
+                radius=0.4,
+                start_angle=cumsum("angles", include_zero=True),
+                end_angle=cumsum("angles"),
+                line_color="white",
+                fill_color="color",
+                legend_field="names",
+                source=self.compute_source,
+            )
+
+            fig2.axis.axis_label = None
+            fig2.axis.visible = False
+            fig2.grid.grid_line_color = None
+            fig2.add_layout(
+                Title(
+                    text="Note: tasks less than 2% of max are not displayed",
+                    text_font_style="italic",
+                ),
+                "below",
+            )
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Name:</b> @names</p>
+                <p><b>Time:</b> @formatted_time</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig2.add_tools(hover)
+            self.wedge_fig = fig2
+            tab2 = Panel(child=fig2, title="Pie Chart")
+
+            self.tabs = Tabs(tabs=[tab1, tab2])
 
     @without_property_validation
     def update(self):
@@ -523,22 +590,33 @@ def update(self):
                 compute_times.items(), key=lambda x: x[1], reverse=True
             )
 
-            compute_colors = list()
-            compute_names = list()
-            compute_time = list()
-            for name, t in compute_times:
-                compute_names.append(name)
-                compute_colors.append(ts_color_of(name))
-                compute_time.append(t)
-
-            self.fig.x_range.factors = compute_names
-            self.fig.title.text = "Compute Time Per Task"
-
-            compute_result = dict(
-                times=compute_time, color=compute_colors, names=compute_names
-            )
+            # keep only time which are 2% of max or greater
+            if compute_times:
+                max_time = compute_times[0][1] * 0.02
+                compute_times = [(n, t) for n, t in compute_times if t > max_time]
+                compute_colors = list()
+                compute_names = list()
+                compute_time = list()
+                total_time = 0
+                for name, t in compute_times:
+                    compute_names.append(name)
+                    compute_colors.append(ts_color_of(name))
+                    compute_time.append(t)
+                    total_time += t
+
+                angles = [t / total_time * 2 * math.pi for t in compute_time]
+
+                self.fig.x_range.factors = compute_names
+
+                compute_result = dict(
+                    angles=angles,
+                    times=compute_time,
+                    color=compute_colors,
+                    names=compute_names,
+                    formatted_time=[format_time(t) for t in compute_time],
+                )
 
-            update(self.compute_source, compute_result)
+                update(self.compute_source, compute_result)
 
 
 class AggregateAction(DashboardComponent):
@@ -557,6 +635,7 @@ def __init__(self, scheduler, **kwargs):
 
             action_data = {
                 "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
                 "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
                 "names": ["transfer", "compute"],
             }
@@ -578,13 +657,12 @@ def __init__(self, scheduler, **kwargs):
                 top="times",
                 width=0.7,
                 color="color",
-                legend_field="names",
             )
 
             fig.y_range.start = 0
             fig.min_border_right = 20
             fig.min_border_bottom = 60
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.axis_label = "Time (s)"
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
@@ -601,7 +679,7 @@ def __init__(self, scheduler, **kwargs):
             hover.tooltips = """
             <div>
                 <p><b>Name:</b> @names</p>
-                <p><b>Time:</b> @times s</p>
+                <p><b>Time:</b> @formatted_time</p>
             </div>
             """
             hover.point_policy = "follow_mouse"
@@ -635,7 +713,12 @@ def update(self):
             self.fig.x_range.factors = agg_names
             self.fig.title.text = "Aggregate Time Per Action"
 
-            action_result = dict(times=agg_time, color=agg_colors, names=agg_names)
+            action_result = dict(
+                times=agg_time,
+                color=agg_colors,
+                names=agg_names,
+                formatted_time=[format_time(t) for t in agg_time],
+            )
 
             update(self.action_source, action_result)
 
@@ -2129,10 +2212,10 @@ def individual_memory_by_key_doc(scheduler, extra, doc):
 
 def individual_compute_time_per_key_doc(scheduler, extra, doc):
     with log_errors():
-        component = ComputerPerKey(scheduler, sizing_mode="stretch_both")
+        component = ComputePerKey(scheduler, sizing_mode="stretch_both")
         component.update()
         add_periodic_callback(doc, component, 500)
-        doc.add_root(component.fig)
+        doc.add_root(component.tabs)
         doc.theme = BOKEH_THEME
 
 


Test patch:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -36,7 +36,7 @@
     ProfileServer,
     MemoryByKey,
     AggregateAction,
-    ComputerPerKey,
+    ComputePerKey,
 )
 from distributed.dashboard import scheduler
 
@@ -741,8 +741,8 @@ async def test_aggregate_action(c, s, a, b):
 
 
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
-async def test_computer_per_key(c, s, a, b):
-    mbk = ComputerPerKey(s)
+async def test_compute_per_key(c, s, a, b):
+    mbk = ComputePerKey(s)
 
     da = pytest.importorskip("dask.array")
     x = (da.ones((20, 20), chunks=(10, 10)) + 1).persist(optimize_graph=False)
@@ -759,8 +759,8 @@ async def test_computer_per_key(c, s, a, b):
     )
     assert response.code == 200
     assert ("sum-aggregate") in mbk.compute_source.data["names"]
-    assert ("inc") in mbk.compute_source.data["names"]
     assert ("add") in mbk.compute_source.data["names"]
+    assert "angles" in mbk.compute_source.data.keys()
 
 
 @gen_cluster(scheduler_kwargs={"http_prefix": "foo-bar", "dashboard": True})