Repository: arviz-devs/arviz

Issue:
plot_compare issue with pWAIC
The plot_compare method seems to add the pWAIC values to the in-sample deviance to get WAIC values regardless of scale (deviance or log). Shouldn't the pWAIC be subtracted in the log scale, where a higher score is better? Otherwise, for example with two models m1 and m2 with the same in-sample deviance of 20: if m1 has pWAIC of 10, m2 has pWAIC of 5 then m1 WAIC is 30 and m2 WAIC is 25 so m1 is preferred. However, with the same in-sample deviance the model with the lower pWAIC should be preferred i.e. m2.

Example from my work:

![image](https://user-images.githubusercontent.com/8579413/87334769-28530e80-c50d-11ea-96c0-6cf2e240e346.png)

I might be getting confused about this and my example isn't well explained, sorry.


Discussion:

Resolving patch:
diff --git a/arviz/plots/backends/bokeh/compareplot.py b/arviz/plots/backends/bokeh/compareplot.py
--- a/arviz/plots/backends/bokeh/compareplot.py
+++ b/arviz/plots/backends/bokeh/compareplot.py
@@ -105,8 +105,16 @@ def plot_compare(
         ax.multi_line(err_xs, err_ys, line_color=plot_kwargs.get("color_ic", "black"))
 
     if insample_dev:
+        scale = comp_df[f"{information_criterion}_scale"][0]
+        p_ic = comp_df[f"p_{information_criterion}"]
+        if scale == "log":
+            correction = p_ic
+        elif scale == "negative_log":
+            correction = -p_ic
+        elif scale == "deviance":
+            correction = -(2 * p_ic)
         ax.circle(
-            comp_df[information_criterion] - (2 * comp_df["p_" + information_criterion]),
+            comp_df[information_criterion] + correction,
             yticks_pos[::2],
             line_color=plot_kwargs.get("color_insample_dev", "black"),
             fill_color=plot_kwargs.get("color_insample_dev", "black"),
diff --git a/arviz/plots/backends/matplotlib/compareplot.py b/arviz/plots/backends/matplotlib/compareplot.py
--- a/arviz/plots/backends/matplotlib/compareplot.py
+++ b/arviz/plots/backends/matplotlib/compareplot.py
@@ -82,8 +82,16 @@ def plot_compare(
         )
 
     if insample_dev:
+        scale = comp_df[f"{information_criterion}_scale"][0]
+        p_ic = comp_df[f"p_{information_criterion}"]
+        if scale == "log":
+            correction = p_ic
+        elif scale == "negative_log":
+            correction = -p_ic
+        elif scale == "deviance":
+            correction = -(2 * p_ic)
         ax.plot(
-            comp_df[information_criterion] - (2 * comp_df["p_" + information_criterion]),
+            comp_df[information_criterion] + correction,
             yticks_pos[::2],
             color=plot_kwargs.get("color_insample_dev", "k"),
             marker=plot_kwargs.get("marker_insample_dev", "o"),


Test patch:
diff --git a/arviz/tests/base_tests/test_plots_bokeh.py b/arviz/tests/base_tests/test_plots_bokeh.py
--- a/arviz/tests/base_tests/test_plots_bokeh.py
+++ b/arviz/tests/base_tests/test_plots_bokeh.py
@@ -326,16 +326,6 @@ def test_plot_compare(models, kwargs):
     assert axes
 
 
-def test_plot_compare_manual(models):
-    """Test compare plot without scale column"""
-    model_compare = compare({"Model 1": models.model_1, "Model 2": models.model_2})
-
-    # remove "scale" column
-    del model_compare["loo_scale"]
-    axes = plot_compare(model_compare, backend="bokeh", show=False)
-    assert axes
-
-
 def test_plot_compare_no_ic(models):
     """Check exception is raised if model_compare doesn't contain a valid information criterion"""
     model_compare = compare({"Model 1": models.model_1, "Model 2": models.model_2})
diff --git a/arviz/tests/base_tests/test_plots_matplotlib.py b/arviz/tests/base_tests/test_plots_matplotlib.py
--- a/arviz/tests/base_tests/test_plots_matplotlib.py
+++ b/arviz/tests/base_tests/test_plots_matplotlib.py
@@ -937,16 +937,6 @@ def test_plot_compare(models, kwargs):
     assert axes
 
 
-def test_plot_compare_manual(models):
-    """Test compare plot without scale column"""
-    model_compare = compare({"Model 1": models.model_1, "Model 2": models.model_2})
-
-    # remove "scale" column
-    del model_compare["loo_scale"]
-    axes = plot_compare(model_compare)
-    assert axes
-
-
 def test_plot_compare_no_ic(models):
     """Check exception is raised if model_compare doesn't contain a valid information criterion"""
     model_compare = compare({"Model 1": models.model_1, "Model 2": models.model_2})