Repository: biolab/orange3

Issue:
Predictions require clicking twice, can't sort by probabilities
#4307, which I just merged, fixes the problem, but -- as I noticed a moment later -- not entirely. Instead of reverting, I'm writing a PR and described another, related problem that already existed before.

### Twice-click

Say I sort by predictions. I click on some data column and the order is incorrect. But then I click again and the order is correct. I keep clicking on data columns and the order is always correct in a first try (which must be why I haven't noticed the problem!).

But then I click on predictions column. I don't get the correct order of predictions. Click again and the order is correct. If you now try sorting by a data column, you have to click twice, again...

### Sorting by probabilities

`PredictionSortProxyModel.setProbInd` is never called.

If I display probabilities for a single class, I would expect to see column sorted by that probability. If I select multiple, I'd like sorting by the first probability, and ties resolved by the next.


Discussion:

Resolving patch:
diff --git a/Orange/base.py b/Orange/base.py
--- a/Orange/base.py
+++ b/Orange/base.py
@@ -413,7 +413,11 @@ def __init__(self, skl_model):
 
     def predict(self, X):
         value = self.skl_model.predict(X)
-        if hasattr(self.skl_model, "predict_proba"):
+        # SVM has probability attribute which defines if method compute probs
+        has_prob_attr = hasattr(self.skl_model, "probability")
+        if (has_prob_attr and self.skl_model.probability
+                or not has_prob_attr
+                and hasattr(self.skl_model, "predict_proba")):
             probs = self.skl_model.predict_proba(X)
             return value, probs
         return value
diff --git a/Orange/classification/svm.py b/Orange/classification/svm.py
--- a/Orange/classification/svm.py
+++ b/Orange/classification/svm.py
@@ -12,13 +12,7 @@
 
 
 class SVMClassifier(SklModel):
-
-    def predict(self, X):
-        value = self.skl_model.predict(X)
-        if self.skl_model.probability:
-            prob = self.skl_model.predict_proba(X)
-            return value, prob
-        return value
+    pass
 
 
 class SVMLearner(SklLearner):
diff --git a/Orange/widgets/evaluate/owpredictions.py b/Orange/widgets/evaluate/owpredictions.py
--- a/Orange/widgets/evaluate/owpredictions.py
+++ b/Orange/widgets/evaluate/owpredictions.py
@@ -1,5 +1,6 @@
 from collections import namedtuple
 from functools import partial
+from operator import itemgetter
 
 import numpy
 from AnyQt.QtWidgets import (
@@ -18,6 +19,7 @@
 from Orange.widgets import gui, settings
 from Orange.widgets.evaluate.utils import (
     ScoreTable, usable_scorers, learner_name, scorer_caller)
+from Orange.widgets.utils.colorpalette import ColorPaletteGenerator
 from Orange.widgets.utils.widgetpreview import WidgetPreview
 from Orange.widgets.widget import OWWidget, Msg, Input, Output
 from Orange.widgets.utils.itemmodels import TableModel
@@ -133,6 +135,11 @@ def set_data(self, data):
             modelproxy.setSourceModel(model)
             self.dataview.setModel(modelproxy)
 
+            self.dataview.model().list_sorted.connect(
+                partial(
+                    self._update_data_sort_order, self.dataview,
+                    self.predictionsview))
+
         self._invalidate_predictions()
 
     @property
@@ -305,6 +312,15 @@ def _non_errored_predictors(self):
         return [p for p in self.predictors.values()
                 if isinstance(p.results, Results)]
 
+    def _reordered_probabilities(self, prediction):
+        cur_values = prediction.predictor.domain.class_var.values
+        new_ind = [self.class_values.index(x) for x in cur_values]
+        probs = prediction.results.unmapped_probabilities
+        new_probs = numpy.full(
+            (probs.shape[0], len(self.class_values)), numpy.nan)
+        new_probs[:, new_ind] = probs
+        return new_probs
+
     def _update_predictions_model(self):
         results = []
         headers = []
@@ -312,7 +328,8 @@ def _update_predictions_model(self):
             values = p.results.unmapped_predicted
             target = p.predictor.domain.class_var
             if target.is_discrete:
-                prob = p.results.unmapped_probabilities
+                # order probabilities in order from Show prob. for
+                prob = self._reordered_probabilities(p)
                 values = [Value(target, v) for v in values]
             else:
                 prob = numpy.zeros((len(values), 0))
@@ -336,11 +353,7 @@ def _update_predictions_model(self):
         # model.rowCount(...), `model.parent`, ... calls)
         hheader.setSectionsClickable(predmodel.rowCount() < 20000)
 
-        self.dataview.horizontalHeader().sectionClicked.connect(
-            partial(
-                self._update_data_sort_order, self.dataview,
-                self.predictionsview))
-        self.predictionsview.horizontalHeader().sectionClicked.connect(
+        self.predictionsview.model().list_sorted.connect(
             partial(
                 self._update_data_sort_order, self.predictionsview,
                 self.dataview))
@@ -364,7 +377,6 @@ def _update_data_sort_order(sort_source_view, sort_dest_view):
             else:
                 sortind = None
 
-            sort_source.setSortIndices(None)
             sort_dest.setSortIndices(sortind)
 
         sort_dest_view.horizontalHeader().setSortIndicatorShown(
@@ -384,14 +396,74 @@ def _reset_order(self):
         self.predictionsview.horizontalHeader().setSortIndicatorShown(False)
         self.dataview.horizontalHeader().setSortIndicatorShown(False)
 
+    def _all_color_values(self):
+        """
+        Return list of colors together with their values from all predictors
+        classes. Colors and values are sorted according to the values order
+        for simpler comparison.
+        """
+        predictors = self._non_errored_predictors()
+        color_values = [
+            list(zip(*sorted(zip(
+                p.predictor.domain.class_var.colors,
+                p.predictor.domain.class_var.values
+            ), key=itemgetter(1))))
+            for p in predictors if p.predictor.domain.class_var.is_discrete
+        ]
+        return color_values if color_values else [([], [])]
+
+    @staticmethod
+    def _colors_match(colors1, values1, color2, values2):
+        """
+        Test whether colors for values match. Colors matches when all
+        values match for shorter list and colors match for shorter list.
+        It is assumed that values will be sorted together with their colors.
+        """
+        shorter_length = min(len(colors1), len(color2))
+        return (values1[:shorter_length] == values2[:shorter_length]
+                and (numpy.array(colors1[:shorter_length]) ==
+                     numpy.array(color2[:shorter_length])).all())
+
+    def _get_colors(self):
+        """
+        Defines colors for values. If colors match in all models use the union
+        otherwise use standard colors.
+        """
+        all_colors_values = self._all_color_values()
+        base_color, base_values = all_colors_values[0]
+        for c, v in all_colors_values[1:]:
+            if not self._colors_match(base_color, base_values, c, v):
+                base_color = []
+                break
+            # replace base_color if longer
+            if len(v) > len(base_color):
+                base_color = c
+                base_values = v
+
+        if len(base_color) != len(self.class_values):
+            return ColorPaletteGenerator.palette(len(self.class_values))
+        # reorder colors to widgets order
+        colors = [None] * len(self.class_values)
+        for c, v in zip(base_color, base_values):
+            colors[self.class_values.index(v)] = c
+        return colors
+
     def _update_prediction_delegate(self):
-        selected = {self.class_values[i] for i in self.selected_classes}
         self._delegates.clear()
+        colors = self._get_colors()
         for col, slot in enumerate(self.predictors.values()):
             target = slot.predictor.domain.class_var
-            shown_probs = () if target.is_continuous else \
-                [i for i, name in enumerate(target.values) if name in selected]
-            delegate = PredictionsItemDelegate(target, shown_probs)
+            shown_probs = (
+                () if target.is_continuous else
+                [val if self.class_values[val] in target.values else None
+                 for val in self.selected_classes]
+            )
+            delegate = PredictionsItemDelegate(
+                None if target.is_continuous else self.class_values,
+                colors,
+                shown_probs,
+                target.format_str if target.is_continuous else None
+            )
             # QAbstractItemView does not take ownership of delegates, so we must
             self._delegates.append(delegate)
             self.predictionsview.setItemDelegateForColumn(col, delegate)
@@ -399,6 +471,8 @@ def _update_prediction_delegate(self):
 
         self.predictionsview.resizeColumnsToContents()
         self._update_spliter()
+        if self.predictionsview.model() is not None:
+            self.predictionsview.model().setProbInd(self.selected_classes)
 
     def _update_spliter(self):
         if not self.data:
@@ -521,26 +595,35 @@ class PredictionsItemDelegate(QStyledItemDelegate):
     """
     A Item Delegate for custom formatting of predictions/probabilities
     """
-    def __init__(self, target, shown_probabilities=(), parent=None, **kwargs):
-        super().__init__(parent, **kwargs)
-        self.target = target
-        self.colors = None if target.is_continuous else \
-                [QColor(*color) for color in target.colors]
+    def __init__(
+            self, class_values, colors, shown_probabilities=(),
+            target_format=None, parent=None,
+    ):
+        super().__init__(parent)
+        self.class_values = class_values  # will be None for continuous
+        self.colors = [QColor(*c) for c in colors]
+        self.target_format = target_format  # target format for cont. vars
         self.shown_probabilities = self.fmt = self.tooltip = None  # set below
         self.setFormat(shown_probabilities)
 
     def setFormat(self, shown_probabilities=()):
         self.shown_probabilities = shown_probabilities
-        target = self.target
-        if target.is_continuous:
-            self.fmt = f"{{value:{target.format_str[1:]}}}"
+        if self.class_values is None:
+            # is continuous class
+            self.fmt = f"{{value:{self.target_format[1:]}}}"
         else:
             self.fmt = " \N{RIGHTWARDS ARROW} ".join(
-                [" : ".join(f"{{dist[{i}]:.2f}}" for i in shown_probabilities)]
+                [" : ".join(f"{{dist[{i}]:.2f}}" if i is not None else "-"
+                            for i in shown_probabilities)]
                 * bool(shown_probabilities)
                 + ["{value!s}"])
-        self.tooltip = "" if not shown_probabilities else \
-            f"p({', '.join(target.values[i] for i in shown_probabilities)})"
+        self.tooltip = ""
+        if shown_probabilities:
+            val = ', '.join(
+                self.class_values[i] if i is not None else "-"
+                for i in shown_probabilities if i is not None
+            )
+            self.tooltip = f"p({val})"
 
     def displayText(self, value, _locale):
         try:
@@ -560,7 +643,7 @@ def helpEvent(self, event, view, option, index):
 
     def initStyleOption(self, option, index):
         super().initStyleOption(option, index)
-        if self.target.is_continuous:
+        if self.class_values is None:
             option.displayAlignment = \
                 (option.displayAlignment & Qt.AlignVertical_Mask) | \
                 Qt.AlignRight
@@ -592,9 +675,6 @@ def paint(self, painter, option, index):
         if dist is None or self.colors is None:
             super().paint(painter, option, index)
             return
-        if not numpy.isfinite(numpy.sum(dist)):
-            super().paint(painter, option, index)
-            return
 
         if option.widget is not None:
             style = option.widget.style()
@@ -656,6 +736,8 @@ def drawDistBar(self, painter, rect, distribution):
         painter.save()
         painter.translate(rect.topLeft())
         for i in self.shown_probabilities:
+            if i is None:
+                continue
             dvalue = distribution[i]
             if not dvalue > 0:  # This also skips nans
                 continue
@@ -670,7 +752,7 @@ class SortProxyModel(QSortFilterProxyModel):
     """
     QSortFilter model used in both TableView and PredictionsView
     """
-    sort_demanded = pyqtSignal()
+    list_sorted = pyqtSignal()
 
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -686,10 +768,7 @@ def setSortIndices(self, indices):
         self.__sortInd = indices
 
         if self.__sortInd is not None:
-            if self.sortColumn() < 0:
-                self.sort(0)  # need some valid sort column
-            else:
-                self.invalidate()
+            self.custom_sort(0)  # need valid order to call lessThan
 
     def lessThan(self, left, right):
         if self.__sortInd is None:
@@ -709,6 +788,27 @@ def lessThan(self, left, right):
     def isSorted(self):
         return self.__sortInd is not None
 
+    def sort(self, n, order=Qt.AscendingOrder):
+        """
+        This sort is called only on click, in other cases we manually call
+        custom_sort
+        """
+        # reset sort - otherwise when same parameters set by lessThan function
+        # clicking on header would not trigger resort
+        self.__sortInd = None
+        self.custom_sort(n, order=order)
+        self.list_sorted.emit()
+
+    def custom_sort(self, n, order=Qt.AscendingOrder):
+        """
+        When sorting is dmanded because of sort change in the other view
+        this sorting is called. It will not reset __sortInd to None.
+        """
+        if self.sortColumn() == n and self.sortOrder() == order:
+            self.invalidate()
+        else:
+            super().sort(n, order)  # need some valid sort column
+
 
 class PredictionsSortProxyModel(SortProxyModel):
     def __init__(self, parent=None):
@@ -718,6 +818,7 @@ def __init__(self, parent=None):
     def setProbInd(self, indices):
         self.__probInd = indices
         self.invalidate()
+        self.list_sorted.emit()
 
     def lessThan(self, left, right):
         if self.isSorted():
@@ -832,7 +933,10 @@ def tool_tip(value):
     filename = "iris.tab"
     iris = Orange.data.Table(filename)
     idom = iris.domain
-    dom = Domain(idom.attributes, DiscreteVariable(idom.class_var.name, idom.class_var.values[:2]))
+    dom = Domain(
+        idom.attributes,
+        DiscreteVariable(idom.class_var.name, idom.class_var.values[1::-1])
+    )
     iris2 = iris[:100].transform(dom)
 
     def pred_error(data, *args, **kwargs):
@@ -842,20 +946,20 @@ def pred_error(data, *args, **kwargs):
     pred_error.name = "To err is human"
 
     if iris.domain.has_discrete_class:
-        predictors = [
+        predictors_ = [
             Orange.classification.SVMLearner(probability=True)(iris2),
             Orange.classification.LogisticRegressionLearner()(iris),
             pred_error
         ]
     elif iris.domain.has_continuous_class:
-        predictors = [
+        predictors_ = [
             Orange.regression.RidgeRegressionLearner(alpha=1.0)(iris),
             Orange.regression.LinearRegressionLearner()(iris),
             pred_error
         ]
     else:
-        predictors = [pred_error]
+        predictors_ = [pred_error]
 
     WidgetPreview(OWPredictions).run(
         set_data=iris2,
-        set_predictor=[(pred, i) for i, pred in enumerate(predictors)])
+        set_predictor=[(pred, i) for i, pred in enumerate(predictors_)])


Test patch:
diff --git a/Orange/widgets/evaluate/tests/test_owpredictions.py b/Orange/widgets/evaluate/tests/test_owpredictions.py
--- a/Orange/widgets/evaluate/tests/test_owpredictions.py
+++ b/Orange/widgets/evaluate/tests/test_owpredictions.py
@@ -4,6 +4,7 @@
 
 import numpy as np
 
+from Orange.base import Model
 from Orange.classification import LogisticRegressionLearner
 from Orange.data.io import TabReader
 from Orange.widgets.tests.base import WidgetTest
@@ -17,6 +18,7 @@
 from Orange.modelling import ConstantLearner, TreeLearner
 from Orange.evaluation import Results
 from Orange.widgets.tests.utils import excepthook_catch
+from Orange.widgets.utils.colorpalette import ColorPaletteGenerator
 
 
 class TestOWPredictions(WidgetTest):
@@ -176,7 +178,7 @@ def test_predictor_fails(self):
         model2 = ConstantLearner()(titanic)
         self.send_signal(self.widget.Inputs.predictors, model2, 2)
 
-    def test_sort(self):
+    def test_sort_matching(self):
         def get_items_order(model):
             n = pred_model.rowCount()
             return [
@@ -222,12 +224,194 @@ def get_items_order(model):
         self.assertListEqual(pred_order, list(range(n)))
         self.assertListEqual(data_order, list(range(n)))
 
+    def test_sort_predictions(self):
+        """
+        Test whether sorting of probabilities by FilterSortProxy is correct.
+        """
+        def get_items_order(model):
+            n = pred_model.rowCount()
+            return [
+                pred_model.mapToSource(model.index(i, 0)).row()
+                for i in range(n)
+            ]
+
+        log_reg_iris = LogisticRegressionLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, log_reg_iris)
+        self.send_signal(self.widget.Inputs.data, self.iris)
+        _, log_reg_probs = log_reg_iris(self.iris, Model.ValueProbs)
+        # here I assume that classes are in order 0, 1, 2 in widget
+
+        pred_model = self.widget.predictionsview.model()
+        pred_model.sort(0)
+        widget_order = get_items_order(pred_model)
+
+        # correct order first sort by probs[:,0] then probs[:,1], ...
+        keys = tuple(
+            log_reg_probs[:, i] for i in range(
+                log_reg_probs.shape[1] - 1, -1, -1))
+        sort_order = np.lexsort(keys)
+        np.testing.assert_array_equal(widget_order, sort_order)
+
     def test_reset_no_data(self):
         """
         Check no error when resetting the view without model and data
         """
         self.widget.reset_button.click()
 
+    def test_colors_same_domain(self):
+        """
+        Test whether the color selection for values is correct.
+        """
+        # pylint: disable=protected-access
+        self.send_signal(self.widget.Inputs.data, self.iris)
+
+        # case 1: one same model
+        predictor_iris = ConstantLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(
+            colors, self.iris.domain.class_var.colors)
+
+        # case 2: two same models
+        predictor_iris1 = ConstantLearner()(self.iris)
+        predictor_iris2 = ConstantLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        # assume that colors have same order since values have same order
+        np.testing.assert_array_equal(
+            colors, self.iris.domain.class_var.colors)
+
+        # case 3: two same models - changed color order
+        idom = self.iris.domain
+        dom = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom.class_var.colors = dom.class_var.colors[::-1]
+        iris2 = self.iris.transform(dom)
+
+        predictor_iris1 = ConstantLearner()(iris2)
+        predictor_iris2 = ConstantLearner()(iris2)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(colors, iris2.domain.class_var.colors)
+
+    def test_colors_diff_domain(self):
+        """
+        Test whether the color selection for values is correct.
+        """
+        # pylint: disable=protected-access
+        self.send_signal(self.widget.Inputs.data, self.iris)
+
+        # case 1: two domains one subset other
+        idom = self.iris.domain
+        dom1 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom2 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values[:2])
+        )
+        iris1 = self.iris[:100].transform(dom1)
+        iris2 = self.iris[:100].transform(dom2)
+
+        predictor_iris1 = ConstantLearner()(iris1)
+        predictor_iris2 = ConstantLearner()(iris2)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(colors, iris1.domain.class_var.colors)
+
+        # case 2: two domains one subset other - different color order
+        idom = self.iris.domain
+        colors = idom.class_var.colors[::-1]
+        dom1 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom2 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values[:2])
+        )
+        dom1.class_var.colors = colors
+        dom2.class_var.colors = colors[:2]
+        iris1 = self.iris[:100].transform(dom1)
+        iris2 = self.iris[:100].transform(dom2)
+
+        predictor_iris1 = ConstantLearner()(iris1)
+        predictor_iris2 = ConstantLearner()(iris2)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(colors, iris1.domain.class_var.colors)
+
+        # case 3: domain color, values miss-match - use default colors
+        idom = self.iris.domain
+        dom1 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom2 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom1.class_var.colors = dom1.class_var.colors[::-1]
+        iris1 = self.iris.transform(dom1)
+        iris2 = self.iris.transform(dom2)
+
+        predictor_iris1 = ConstantLearner()(iris1)
+        predictor_iris2 = ConstantLearner()(iris2)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(colors, ColorPaletteGenerator.palette(3))
+
+        # case 4: two domains different values order, matching colors
+        idom = self.iris.domain
+        # this way we know that default colors are not used
+        colors = ColorPaletteGenerator.palette(5)[2:]
+        dom1 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values)
+        )
+        dom2 = Domain(
+            idom.attributes,
+            DiscreteVariable(idom.class_var.name, idom.class_var.values[::-1])
+        )
+        dom1.class_var.colors = colors
+        dom2.class_var.colors = colors[::-1]  # colors mixed same than values
+        iris1 = self.iris[:100].transform(dom1)
+        iris2 = self.iris[:100].transform(dom2)
+
+        predictor_iris1 = ConstantLearner()(iris1)
+        predictor_iris2 = ConstantLearner()(iris2)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris1)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris2, 1)
+        colors = self.widget._get_colors()
+        np.testing.assert_array_equal(colors, iris1.domain.class_var.colors)
+
+    def test_colors_continuous(self):
+        """
+        When only continuous variables in predictor no color should be selected
+        we do not work with classes.
+        When we add one classifier there should be colors.
+        """
+        # pylint: disable=protected-access
+        data = Table("housing")
+        cl_data = ConstantLearner()(data)
+        self.send_signal(self.widget.Inputs.predictors, cl_data)
+        self.send_signal(self.widget.Inputs.data, data)
+        colors = self.widget._get_colors()
+        self.assertListEqual([], colors)
+
+        predictor_iris = ConstantLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, predictor_iris, 1)
+        colors = self.widget._get_colors()
+        self.assertEqual(3, len(colors))
+
 
 if __name__ == "__main__":
     import unittest