Repository: biolab/orange3

Issue:
Evaluation of Trained Models on New Data
##### Usecase scenario
Say I once trained and saved a classifier. Later, when some more labeled data arrives, I would like to asses the performance of my trained model on the new data (like with Test & Score). Hence I load the model and new data set but can only pass them to `Predictions`. What's missing is the calculation of various metrics, like we have it in Test & Score, from the predictions on new data.

Similarly, this would be useful for training a classifier and evaluating it on the held out data set. Though, this could be done with `Test & Score` to obtain the score, the user has no access to the classifier trained in `Test & Score` and is required to train an other model if he want's to save it.

#### Possible approaches
* `Test & Score` could also accept classifiers and not only learners and would only offer options `Test on test data` and `Test on train data`. Though, this somehow conflicts with the `Test` part of the widget name.
* `Test & Score` could accept predictions (evaluation results) from the `Predictions` widget and it would only calculate and show scores. Same concern as above.
* A new widget for calculating metrics that accepts trained models and new data. Some functionality would probably be duplicated with `Test & Score`.


Discussion:
Adding this to Test & Score would complicate the widget. Also, many user connect File -> Learner -> Test & Score, where in fact they want to test a learner and not a model. If Test & Score accepted models, too, this would be at least ambiguous.

Accepting evaluation results would also complicate the widget - it is essentially a different widget then, because the left part becomes irrelevant.

We decided that a good solution could be to vertically split the left part of Predictions widget and show table with CA, AUC in the upper part.

Of course, we won't copy the code from Test & Score, but move the code into a separate module to be used by both, Test & Score and Predictions.
Like this. Any objections?

<img width="728" alt="Screenshot 2019-06-06 at 16 09 54" src="https://user-images.githubusercontent.com/2387315/59039730-a8706480-8875-11e9-9637-72f2b1839c33.png">

(Splitter is adjustable.)

Resolving patch:
diff --git a/Orange/widgets/evaluate/owpredictions.py b/Orange/widgets/evaluate/owpredictions.py
--- a/Orange/widgets/evaluate/owpredictions.py
+++ b/Orange/widgets/evaluate/owpredictions.py
@@ -1,21 +1,15 @@
-"""
-Predictions widget
-
-"""
 from collections import OrderedDict, namedtuple
 
 import numpy
 from AnyQt.QtWidgets import (
     QTableView, QListWidget, QSplitter, QStyledItemDelegate,
     QToolTip, QAbstractItemView, QStyleOptionViewItem, QStyle,
-    QApplication,
-)
-from AnyQt.QtGui import QPainter, QHelpEvent
+    QApplication)
+from AnyQt.QtGui import (
+    QPainter, QHelpEvent, QStandardItem, QBrush, QPen, QColor)
 from AnyQt.QtCore import (
-    Qt, QSize, QModelIndex, QAbstractTableModel, QSortFilterProxyModel,
-    QLocale, pyqtSlot as Slot
-)
-from AnyQt import QtCore, QtGui
+    Qt, QSize, QRect, QRectF, QPoint, QLocale, pyqtSlot as Slot,
+    QModelIndex, QAbstractTableModel, QSortFilterProxyModel)
 
 import Orange
 import Orange.evaluation
@@ -24,6 +18,8 @@
 from Orange.data import ContinuousVariable, DiscreteVariable, Value
 from Orange.data.table import DomainTransformationError
 from Orange.widgets import gui, settings
+from Orange.widgets.evaluate.utils import (
+    ScoreTable, usable_scorers, learner_name, scorer_caller)
 from Orange.widgets.utils.widgetpreview import WidgetPreview
 from Orange.widgets.widget import OWWidget, Msg, Input, Output
 from Orange.widgets.utils.itemmodels import TableModel
@@ -55,9 +51,8 @@ class Inputs:
 
     class Outputs:
         predictions = Output("Predictions", Orange.data.Table)
-        evaluation_results = Output("Evaluation Results",
-                                    Orange.evaluation.Results,
-                                    dynamic=False)
+        evaluation_results = Output(
+            "Evaluation Results", Orange.evaluation.Results, dynamic=False)
 
     class Warning(OWWidget.Warning):
         empty_data = Msg("Empty dataset")
@@ -65,12 +60,16 @@ class Warning(OWWidget.Warning):
     class Error(OWWidget.Error):
         predictor_failed = \
             Msg("One or more predictors failed (see more...)\n{}")
+        scorer_failed = \
+            Msg("One or more scorers failed (see more...)\n{}")
         predictors_target_mismatch = \
             Msg("Predictors do not have the same target.")
         data_target_mismatch = \
             Msg("Data does not have the same target as predictors.")
 
     settingsHandler = settings.ClassValuesContextHandler()
+    score_table = settings.SettingProvider(ScoreTable)
+
     #: Display the full input dataset or only the target variable columns (if
     #: available)
     show_attrs = settings.Setting(True)
@@ -141,6 +140,9 @@ def __init__(self):
 
         gui.rubber(self.controlArea)
 
+        self.vsplitter = QSplitter(
+            orientation=Qt.Vertical, childrenCollapsible=True, handleWidth=2)
+
         self.splitter = QSplitter(
             orientation=Qt.Horizontal,
             childrenCollapsible=False,
@@ -181,13 +183,18 @@ def __init__(self):
         self.splitter.addWidget(self.predictionsview)
         self.splitter.addWidget(self.dataview)
 
-        self.mainArea.layout().addWidget(self.splitter)
+        self.score_table = ScoreTable(self)
+        self.vsplitter.addWidget(self.splitter)
+        self.vsplitter.addWidget(self.score_table.view)
+        self.vsplitter.setStretchFactor(0, 5)
+        self.vsplitter.setStretchFactor(1, 1)
+        self.mainArea.layout().addWidget(self.vsplitter)
 
     @Inputs.data
     @check_sql_input
     def set_data(self, data):
         """Set the input dataset"""
-        if data is not None and not len(data):
+        if data is not None and not data:
             data = None
             self.Warning.empty_data()
         else:
@@ -209,6 +216,7 @@ def set_data(self, data):
 
         self._invalidate_predictions()
 
+    # pylint: disable=redefined-builtin
     @Inputs.predictors
     def set_predictor(self, predictor=None, id=None):
         if id in self.predictors:
@@ -221,7 +229,7 @@ def set_predictor(self, predictor=None, id=None):
             self.predictors[id] = \
                 PredictorSlot(predictor, predictor.name, None)
 
-    def set_class_var(self):
+    def _set_class_var(self):
         pred_classes = set(pred.predictor.domain.class_var
                            for pred in self.predictors.values())
         self.Error.predictors_target_mismatch.clear()
@@ -250,9 +258,9 @@ def set_class_var(self):
             self.selected_classes = []
 
     def handleNewSignals(self):
-        self.set_class_var()
-        if self.data is not None:
-            self._call_predictors()
+        self._set_class_var()
+        self._call_predictors()
+        self._update_scores()
         self._update_predictions_model()
         self._update_prediction_delegate()
         self._set_errors()
@@ -260,23 +268,69 @@ def handleNewSignals(self):
         self.commit()
 
     def _call_predictors(self):
+        if not self.data:
+            return
+        for inputid, slot in self.predictors.items():
+            if slot.results is not None \
+                    and not isinstance(slot.results, str) \
+                    and not numpy.isnan(slot.results.predicted[0]).all():
+                continue
+            try:
+                pred, prob = self.predict(slot.predictor, self.data)
+            except (ValueError, DomainTransformationError) as err:
+                results = "{}: {}".format(slot.predictor.name, err)
+            else:
+                results = Orange.evaluation.Results()
+                results.data = self.data
+                results.domain = self.data.domain
+                results.row_indices = numpy.arange(len(self.data))
+                results.folds = (Ellipsis, )
+                results.actual = self.data.Y
+                results.predicted = pred.reshape((1, len(self.data)))
+                results.probabilities = prob.reshape((1, ) + prob.shape)
+            self.predictors[inputid] = slot._replace(results=results)
+
+    def _update_scores(self):
+        model = self.score_table.model
+        model.clear()
+
+        if self.data is None or self.data.domain.class_var is None:
+            scorers = []
+        else:
+            scorers = usable_scorers(self.data.domain.class_var)
+        self.score_table.update_header(scorers)
+
+        errors = []
         for inputid, pred in self.predictors.items():
-            if pred.results is None \
-                    or isinstance(pred.results, str) \
-                    or numpy.isnan(pred.results[0]).all():
-                try:
-                    results = self.predict(pred.predictor, self.data)
-                except (ValueError, DomainTransformationError) as err:
-                    results = "{}: {}".format(pred.predictor.name, err)
-                self.predictors[inputid] = pred._replace(results=results)
+            name = learner_name(pred.predictor)
+            head = QStandardItem(name)
+            #            head.setData(key, Qt.UserRole)
+            row = [head]
+            results = self.predictors[inputid].results
+            if isinstance(results, str):
+                head.setToolTip(results)
+                head.setText("{} (error)".format(name))
+                head.setForeground(QBrush(Qt.red))
+            else:
+                for scorer in scorers:
+                    item = QStandardItem()
+                    try:
+                        score = scorer_caller(scorer, results)()[0]
+                        item.setText(f"{score:.3f}")
+                    except Exception as exc:  # pylint: disable=broad-except
+                        item.setToolTip(str(exc))
+                        if scorer.name in self.score_table.shown_scores:
+                            errors.append(str(exc))
+                    row.append(item)
+            self.score_table.model.appendRow(row)
+        self.Error.scorer_failed("\n".join(errors), shown=bool(errors))
 
     def _set_errors(self):
+        # Not all predictors are run every time, so errors can't be collected
+        # in _call_predictors
         errors = "\n".join(p.results for p in self.predictors.values()
                            if isinstance(p.results, str))
-        if errors:
-            self.Error.predictor_failed(errors)
-        else:
-            self.Error.predictor_failed.clear()
+        self.Error.predictor_failed(errors, shown=bool(errors))
 
     def _update_info(self):
         info = []
@@ -311,8 +365,7 @@ def _invalidate_predictions(self):
             self.predictors[inputid] = pred._replace(results=None)
 
     def _valid_predictors(self):
-        if self.class_var is not None and \
-                self.data is not None:
+        if self.class_var is not None and self.data is not None:
             return [p for p in self.predictors.values()
                     if p.results is not None and not isinstance(p.results, str)]
         else:
@@ -325,15 +378,21 @@ def _update_predictions_model(self):
             results = []
             class_var = self.class_var
             for p in slots:
-                values, prob = p.results
+                if isinstance(p.results, str):
+                    continue
+                values = p.results.predicted[0]
                 if self.class_var.is_discrete:
                     # if values were added to class_var between building the
                     # model and predicting, add zeros for new class values,
                     # which are always at the end
+                    prob = p.results.probabilities[0]
                     prob = numpy.c_[
                         prob,
-                        numpy.zeros((prob.shape[0], len(class_var.values) - prob.shape[1]))]
+                        numpy.zeros((prob.shape[0],
+                                     len(class_var.values) - prob.shape[1]))]
                     values = [Value(class_var, v) for v in values]
+                else:
+                    prob = numpy.zeros((len(values), 0))
                 results.append((values, prob))
             results = list(zip(*(zip(*res) for res in results)))
             headers = [p.name for p in slots]
@@ -417,7 +476,7 @@ def _update_prediction_delegate(self):
         self._update_spliter()
 
     def _setup_delegate_discrete(self, delegate):
-        colors = [QtGui.QColor(*rgb) for rgb in self.class_var.colors]
+        colors = [QColor(*rgb) for rgb in self.class_var.colors]
         fmt = []
         if self.show_probabilities:
             fmt.append(" : ".join("{{dist[{}]:.2f}}".format(i)
@@ -458,16 +517,15 @@ def _commit_evaluation_results(self):
         class_var = self.class_var
         nanmask = numpy.isnan(self.data.get_column_view(class_var)[0])
         data = self.data[~nanmask]
-        N = len(data)
         results = Orange.evaluation.Results(data, store_data=True)
         results.folds = None
-        results.row_indices = numpy.arange(N)
+        results.row_indices = numpy.arange(len(data))
         results.actual = data.Y.ravel()
         results.predicted = numpy.vstack(
-            tuple(p.results[0][~nanmask] for p in slots))
+            tuple(p.results.predicted[0][~nanmask] for p in slots))
         if class_var and class_var.is_discrete:
             results.probabilities = numpy.array(
-                [p.results[1][~nanmask] for p in slots])
+                [p.results.probabilities[0][~nanmask] for p in slots])
         results.learner_names = [p.name for p in slots]
         self.Outputs.evaluation_results.send(results)
 
@@ -500,18 +558,19 @@ def _classification_output_columns(self):
         if self.output_predictions:
             newmetas += [DiscreteVariable(name=p.name, values=self.class_values)
                          for p in slots]
-            newcolumns += [p.results[0].reshape((-1, 1)) for p in slots]
+            newcolumns += [p.results.predicted[0].reshape((-1, 1))
+                           for p in slots]
 
         if self.output_probabilities:
             newmetas += [ContinuousVariable(name="%s (%s)" % (p.name, value))
                          for p in slots for value in self.class_values]
-            newcolumns += [p.results[1] for p in slots]
+            newcolumns += [p.results.probabilities[0] for p in slots]
         return newmetas, newcolumns
 
     def _regression_output_columns(self):
         slots = self._valid_predictors()
         newmetas = [ContinuousVariable(name=p.name) for p in slots]
-        newcolumns = [p.results[0].reshape((-1, 1)) for p in slots]
+        newcolumns = [p.results.predicted[0].reshape((-1, 1)) for p in slots]
         return newmetas, newcolumns
 
     def send_report(self):
@@ -557,8 +616,9 @@ def predict(cls, predictor, data):
         if class_var:
             if class_var.is_discrete:
                 return cls.predict_discrete(predictor, data)
-            elif class_var.is_continuous:
+            else:
                 return cls.predict_continuous(predictor, data)
+        return None
 
     @staticmethod
     def predict_discrete(predictor, data):
@@ -567,7 +627,7 @@ def predict_discrete(predictor, data):
     @staticmethod
     def predict_continuous(predictor, data):
         values = predictor(data, Model.Value)
-        return values, [None] * len(data)
+        return values, numpy.zeros((len(data), 0))
 
 
 class PredictionsItemDelegate(QStyledItemDelegate):
@@ -587,12 +647,12 @@ def setFormat(self, fmt):
     def setColors(self, colortable):
         if colortable is not None:
             colortable = list(colortable)
-            if not all(isinstance(c, QtGui.QColor) for c in colortable):
+            if not all(isinstance(c, QColor) for c in colortable):
                 raise TypeError
 
         self.__colors = colortable
 
-    def displayText(self, value, locale):
+    def displayText(self, value, _locale):
         try:
             value, dist = value
         except ValueError:
@@ -642,7 +702,8 @@ def sizeHint(self, option, index):
         height = sh.height() + metrics.leading() + 2 * margin
         return QSize(sh.width(), height)
 
-    def distribution(self, index):
+    @staticmethod
+    def distribution(index):
         value = index.data(Qt.DisplayRole)
         if isinstance(value, tuple) and len(value) == 2:
             _, dist = value
@@ -653,9 +714,11 @@ def distribution(self, index):
     def paint(self, painter, option, index):
         dist = self.distribution(index)
         if dist is None or self.__colors is None:
-            return super().paint(painter, option, index)
+            super().paint(painter, option, index)
+            return
         if not numpy.isfinite(numpy.sum(dist)):
-            return super().paint(painter, option, index)
+            super().paint(painter, option, index)
+            return
 
         nvalues = len(dist)
         if len(self.__colors) < nvalues:
@@ -706,14 +769,13 @@ def paint(self, painter, option, index):
             color = option.palette.highlightedText().color()
         else:
             color = option.palette.text().color()
-        painter.setPen(QtGui.QPen(color))
+        painter.setPen(QPen(color))
 
         textrect = textrect.adjusted(0, 0, 0, -distheight - spacing)
-        distrect = QtCore.QRect(
-            textrect.bottomLeft() + QtCore.QPoint(0, spacing),
-            QtCore.QSize(rect.width(), distheight)
-        )
-        painter.setPen(QtGui.QPen(Qt.lightGray, 0.3))
+        distrect = QRect(
+            textrect.bottomLeft() + QPoint(0, spacing),
+            QSize(rect.width(), distheight))
+        painter.setPen(QPen(Qt.lightGray, 0.3))
         drawDistBar(painter, distrect, dist, colors)
         painter.restore()
         if text:
@@ -727,9 +789,9 @@ def drawDistBar(painter, rect, distribution, colortable):
     Parameters
     ----------
     painter : QtGui.QPainter
-    rect : QtCore.QRect
+    rect : QRect
     distribution : numpy.ndarray
-    colortable : List[QtGui.QColor]
+    colortable : List[QColor]
     """
     # assert numpy.isclose(numpy.sum(distribution), 1.0)
     # assert numpy.all(distribution >= 0)
@@ -739,8 +801,7 @@ def drawDistBar(painter, rect, distribution, colortable):
         if dvalue and numpy.isfinite(dvalue):
             painter.setBrush(color)
             width = rect.width() * dvalue
-            painter.drawRoundedRect(
-                QtCore.QRectF(0, 0, width, rect.height()), 1, 2)
+            painter.drawRoundedRect(QRectF(0, 0, width, rect.height()), 1, 2)
             painter.translate(width, 0.0)
     painter.restore()
 
@@ -797,7 +858,7 @@ def _value(self, index):
         return self._table[row][column]
 
     def data(self, index, role=Qt.DisplayRole):
-        if role == Qt.DisplayRole or role == Qt.EditRole:
+        if role in (Qt.DisplayRole, Qt.EditRole):
             return self._value(index)
         else:
             return None
@@ -910,29 +971,29 @@ def tool_tip(value):
 
 if __name__ == "__main__":  # pragma: no cover
     filename = "iris.tab"
-    data = Orange.data.Table(filename)
+    iris = Orange.data.Table(filename)
 
     def pred_error(data, *args, **kwargs):
         raise ValueError
 
-    pred_error.domain = data.domain
+    pred_error.domain = iris.domain
     pred_error.name = "To err is human"
 
-    if data.domain.has_discrete_class:
+    if iris.domain.has_discrete_class:
         predictors = [
-            Orange.classification.SVMLearner(probability=True)(data),
-            Orange.classification.LogisticRegressionLearner()(data),
+            Orange.classification.SVMLearner(probability=True)(iris),
+            Orange.classification.LogisticRegressionLearner()(iris),
             pred_error
         ]
-    elif data.domain.has_continuous_class:
+    elif iris.domain.has_continuous_class:
         predictors = [
-            Orange.regression.RidgeRegressionLearner(alpha=1.0)(data),
-            Orange.regression.LinearRegressionLearner()(data),
+            Orange.regression.RidgeRegressionLearner(alpha=1.0)(iris),
+            Orange.regression.LinearRegressionLearner()(iris),
             pred_error
         ]
     else:
         predictors = [pred_error]
 
     WidgetPreview(OWPredictions).run(
-        set_data=data,
+        set_data=iris,
         set_predictor=[(pred, i) for i, pred in enumerate(predictors)])
diff --git a/Orange/widgets/evaluate/owtestlearners.py b/Orange/widgets/evaluate/owtestlearners.py
--- a/Orange/widgets/evaluate/owtestlearners.py
+++ b/Orange/widgets/evaluate/owtestlearners.py
@@ -1,7 +1,5 @@
 # pylint doesn't understand the Settings magic
 # pylint: disable=invalid-sequence-index
-import warnings
-from itertools import chain
 import abc
 import enum
 import logging
@@ -17,23 +15,23 @@
 import numpy as np
 
 from AnyQt import QtGui
-from AnyQt.QtWidgets import QHeaderView, QStyledItemDelegate, QMenu
-from AnyQt.QtGui import QStandardItemModel, QStandardItem
+from AnyQt.QtGui import QStandardItem
 from AnyQt.QtCore import Qt, QSize, QThread, QMetaObject, Q_ARG
 from AnyQt.QtCore import pyqtSlot as Slot
-from sklearn.exceptions import UndefinedMetricWarning
 
 from Orange.base import Learner
 import Orange.classification
-from Orange.data import Table, DiscreteVariable, ContinuousVariable
+from Orange.data import Table, DiscreteVariable
 from Orange.data.table import DomainTransformationError
 from Orange.data.filter import HasClass
 from Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT
 import Orange.evaluation
-from Orange.evaluation import scoring, Results
+from Orange.evaluation import Results
 from Orange.preprocess.preprocess import Preprocess
 import Orange.regression
 from Orange.widgets import gui, settings, widget
+from Orange.widgets.evaluate.utils import \
+    usable_scorers, ScoreTable, learner_name, scorer_caller
 from Orange.widgets.utils.itemmodels import DomainModel
 from Orange.widgets.utils.widgetpreview import WidgetPreview
 from Orange.widgets.widget import OWWidget, Msg, Input, Output
@@ -50,12 +48,6 @@
 )
 
 
-class ItemDelegate(QStyledItemDelegate):
-    def sizeHint(self, *args):
-        size = super().sizeHint(*args)
-        return QSize(size.width(), size.height() + 6)
-
-
 class Try(abc.ABC):
     """Try to walk in a Turing tar pit."""
 
@@ -156,6 +148,7 @@ class Outputs:
             "click_header")]
 
     settingsHandler = settings.PerfectDomainContextHandler()
+    score_table = settings.SettingProvider(ScoreTable)
 
     #: Resampling/testing types
     KFold, FeatureFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest \
@@ -186,13 +179,6 @@ class Outputs:
     TARGET_AVERAGE = "(Average over classes)"
     class_selection = settings.ContextSetting(TARGET_AVERAGE)
 
-    BUILTIN_ORDER = {
-        DiscreteVariable: ("AUC", "CA", "F1", "Precision", "Recall"),
-        ContinuousVariable: ("MSE", "RMSE", "MAE", "R2")}
-
-    shown_scores = \
-        settings.Setting(set(chain(*BUILTIN_ORDER.values())))
-
     class Error(OWWidget.Error):
         train_data_empty = Msg("Train dataset is empty.")
         test_data_empty = Msg("Test dataset is empty.")
@@ -291,25 +277,11 @@ def __init__(self):
             contentsLength=8)
 
         gui.rubber(self.controlArea)
-
-        self.view = gui.TableView(
-            wordWrap=True,
-            editTriggers=gui.TableView.NoEditTriggers
-        )
-        header = self.view.horizontalHeader()
-        header.setSectionResizeMode(QHeaderView.ResizeToContents)
-        header.setDefaultAlignment(Qt.AlignCenter)
-        header.setStretchLastSection(False)
-        header.setContextMenuPolicy(Qt.CustomContextMenu)
-        header.customContextMenuRequested.connect(self.show_column_chooser)
-
-        self.result_model = QStandardItemModel(self)
-        self.result_model.setHorizontalHeaderLabels(["Method"])
-        self.view.setModel(self.result_model)
-        self.view.setItemDelegate(ItemDelegate())
+        self.score_table = ScoreTable(self)
+        self.score_table.shownScoresChanged.connect(self.update_stats_model)
 
         box = gui.vBox(self.mainArea, "Evaluation Results")
-        box.layout().addWidget(self.view)
+        box.layout().addWidget(self.score_table.view)
 
     def sizeHint(self):
         return QSize(780, 1)
@@ -466,14 +438,7 @@ def _update_scorers(self):
         if self.data is None or self.data.domain.class_var is None:
             self.scorers = []
             return
-        class_var = self.data and self.data.domain.class_var
-        order = {name: i
-                 for i, name in enumerate(self.BUILTIN_ORDER[type(class_var)])}
-        # 'abstract' is retrieved from __dict__ to avoid inheriting
-        usable = (cls for cls in scoring.Score.registry.values()
-                  if cls.is_scalar and not cls.__dict__.get("abstract")
-                  and isinstance(class_var, cls.class_types))
-        self.scorers = sorted(usable, key=lambda cls: order.get(cls.name, 99))
+        self.scorers = usable_scorers(self.data.domain.class_var)
 
     @Inputs.preprocessor
     def set_preprocessor(self, preproc):
@@ -486,8 +451,8 @@ def set_preprocessor(self, preproc):
     def handleNewSignals(self):
         """Reimplemented from OWWidget.handleNewSignals."""
         self._update_class_selection()
-        self._update_header()
-        self._update_stats_model()
+        self.score_table.update_header(self.scorers)
+        self.update_stats_model()
         if self.__needupdate:
             self.__update()
 
@@ -507,32 +472,11 @@ def _param_changed(self):
         self._invalidate()
         self.__update()
 
-    def _update_header(self):
-        # Set the correct horizontal header labels on the results_model.
-        model = self.result_model
-        model.setColumnCount(1 + len(self.scorers))
-        for col, score in enumerate(self.scorers):
-            item = QStandardItem(score.name)
-            item.setToolTip(score.long_name)
-            model.setHorizontalHeaderItem(col + 1, item)
-        self._update_shown_columns()
-
-    def _update_shown_columns(self):
-        # pylint doesn't know that self.shown_scores is a set, not a Setting
-        # pylint: disable=unsupported-membership-test
-        model = self.result_model
-        header = self.view.horizontalHeader()
-        for section in range(1, model.columnCount()):
-            col_name = model.horizontalHeaderItem(section).data(Qt.DisplayRole)
-            header.setSectionHidden(section, col_name not in self.shown_scores)
-        self.view.resizeColumnsToContents()
-        self._update_stats_model()  # updates warnings for displayed columns
-
-    def _update_stats_model(self):
+    def update_stats_model(self):
         # Update the results_model with up to date scores.
         # Note: The target class specific scores (if requested) are
         # computed as needed in this method.
-        model = self.view.model()
+        model = self.score_table.model
         # clear the table model, but preserving the header labels
         for r in reversed(range(model.rowCount())):
             model.takeRow(r)
@@ -591,14 +535,14 @@ def _update_stats_model(self):
                         item.setText("{:.3f}".format(stat.value[0]))
                     else:
                         item.setToolTip(str(stat.exception))
-                        if scorer.name in self.shown_scores:
+                        if scorer.name in self.score_table.shown_scores:
                             has_missing_scores = True
                     row.append(item)
 
             model.appendRow(row)
 
         # Resort rows based on current sorting
-        header = self.view.horizontalHeader()
+        header = self.score_table.view.horizontalHeader()
         model.sort(
             header.sortIndicatorSection(),
             header.sortIndicatorOrder()
@@ -629,7 +573,7 @@ def _update_class_selection(self):
             self.cbox.setVisible(False)
 
     def _on_target_class_changed(self):
-        self._update_stats_model()
+        self.update_stats_model()
 
     def _invalidate(self, which=None):
         self.fold_feature_selected = \
@@ -639,7 +583,7 @@ def _invalidate(self, which=None):
         if which is None:
             which = self.learners.keys()
 
-        model = self.view.model()
+        model = self.score_table.model
         statmodelkeys = [model.item(row, 0).data(Qt.UserRole)
                          for row in range(model.rowCount())]
 
@@ -657,27 +601,6 @@ def _invalidate(self, which=None):
 
         self.__needupdate = True
 
-    def show_column_chooser(self, pos):
-        # pylint doesn't know that self.shown_scores is a set, not a Setting
-        # pylint: disable=unsupported-membership-test
-        def update(col_name, checked):
-            if checked:
-                self.shown_scores.add(col_name)
-            else:
-                self.shown_scores.remove(col_name)
-            self._update_shown_columns()
-
-        menu = QMenu()
-        model = self.result_model
-        header = self.view.horizontalHeader()
-        for section in range(1, model.columnCount()):
-            col_name = model.horizontalHeaderItem(section).data(Qt.DisplayRole)
-            action = menu.addAction(col_name)
-            action.setCheckable(True)
-            action.setChecked(col_name in self.shown_scores)
-            action.triggered.connect(partial(update, col_name))
-        menu.exec(header.mapToGlobal(pos))
-
     def commit(self):
         """
         Commit the results to output.
@@ -728,7 +651,7 @@ def send_report(self):
             items += [("Target class", self.class_selection.strip("()"))]
         if items:
             self.report_items("Settings", items)
-        self.report_table("Scores", self.view)
+        self.report_table("Scores", self.score_table.view)
 
     @classmethod
     def migrate_settings(cls, settings_, version):
@@ -864,7 +787,7 @@ def replace_learners(evalfunc, *args, **kwargs):
         self.__submit(test_f)
 
     def __submit(self, testfunc):
-        # type: (Callable[[Callable[float]], Results]) -> None
+        # type: (Callable[[Callable[[float], None]], Results]) -> None
         """
         Submit a testing function for evaluation
 
@@ -953,8 +876,8 @@ def __task_complete(self, task):
             self.learners[key] = \
                 self.learners[key]._replace(results=result, stats=stats)
 
-        self._update_header()
-        self._update_stats_model()
+        self.score_table.update_header(self.scorers)
+        self.update_stats_model()
 
         self.commit()
 
@@ -974,22 +897,6 @@ def onDeleteWidget(self):
         super().onDeleteWidget()
 
 
-def scorer_caller(scorer, ovr_results, target=None):
-    def thunked():
-        with warnings.catch_warnings():
-            # F-score and Precision return 0 for labels with no predicted
-            # samples. We're OK with that.
-            warnings.filterwarnings(
-                "ignore", "((F-score|Precision)) is ill-defined.*",
-                UndefinedMetricWarning)
-            if scorer.is_binary:
-                return scorer(ovr_results, target=target, average='weighted')
-            else:
-                return scorer(ovr_results)
-
-    return thunked
-
-
 class UserInterrupt(BaseException):
     """
     A BaseException subclass used for cooperative task/thread cancellation
@@ -997,12 +904,6 @@ class UserInterrupt(BaseException):
     pass
 
 
-def learner_name(learner):
-    """Return the value of `learner.name` if it exists, or the learner's type
-    name otherwise"""
-    return getattr(learner, "name", type(learner).__name__)
-
-
 def results_add_by_model(x, y):
     def is_empty(res):
         return (getattr(res, "models", None) is None
@@ -1083,7 +984,7 @@ class Task:
     #: True if the task was cancelled
     cancelled = False   # type: bool
     #: A function to call. Filled by the client.
-    func = ...          # type: Callable[Callable[float], Results]
+    func = ...          # type: Callable[[Callable[[float], None]], Results]
 
     def cancel(self):
         """
diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py
--- a/Orange/widgets/evaluate/utils.py
+++ b/Orange/widgets/evaluate/utils.py
@@ -1,4 +1,19 @@
-import numpy
+import warnings
+from functools import partial
+from itertools import chain
+
+import numpy as np
+
+from AnyQt.QtWidgets import QHeaderView, QStyledItemDelegate, QMenu
+from AnyQt.QtGui import QStandardItemModel, QStandardItem
+from AnyQt.QtCore import Qt, QSize, QObject, pyqtSignal as Signal
+from sklearn.exceptions import UndefinedMetricWarning
+
+from Orange.data import Variable, DiscreteVariable, ContinuousVariable
+from Orange.evaluation import scoring
+from Orange.widgets import gui
+from Orange.widgets.gui import OWComponent
+from Orange.widgets.settings import Setting
 
 
 def check_results_adequacy(results, error_group, check_nan=True):
@@ -6,7 +21,7 @@ def check_results_adequacy(results, error_group, check_nan=True):
     error_group.invalid_results.clear()
 
     def anynan(a):
-        return numpy.any(numpy.isnan(a))
+        return np.any(np.isnan(a))
 
     if results is None:
         return None
@@ -25,6 +40,7 @@ def anynan(a):
     else:
         return results
 
+
 def results_for_preview(data_name=""):
     from Orange.data import Table
     from Orange.evaluation import CrossValidation
@@ -43,3 +59,113 @@ def results_for_preview(data_name=""):
     )
     results.learner_names = ["LR l2", "LR l1", "SVM", "Nu SVM"]
     return results
+
+
+BUILTIN_SCORERS_ORDER = {
+    DiscreteVariable: ("AUC", "CA", "F1", "Precision", "Recall"),
+    ContinuousVariable: ("MSE", "RMSE", "MAE", "R2")}
+
+
+def learner_name(learner):
+    """Return the value of `learner.name` if it exists, or the learner's type
+    name otherwise"""
+    return getattr(learner, "name", type(learner).__name__)
+
+
+def usable_scorers(target: Variable):
+    order = {name: i
+             for i, name in enumerate(BUILTIN_SCORERS_ORDER[type(target)])}
+    # 'abstract' is retrieved from __dict__ to avoid inheriting
+    usable = (cls for cls in scoring.Score.registry.values()
+              if cls.is_scalar and not cls.__dict__.get("abstract")
+              and isinstance(target, cls.class_types))
+    return sorted(usable, key=lambda cls: order.get(cls.name, 99))
+
+
+def scorer_caller(scorer, ovr_results, target=None):
+    def thunked():
+        with warnings.catch_warnings():
+            # F-score and Precision return 0 for labels with no predicted
+            # samples. We're OK with that.
+            warnings.filterwarnings(
+                "ignore", "((F-score|Precision)) is ill-defined.*",
+                UndefinedMetricWarning)
+            if scorer.is_binary:
+                return scorer(ovr_results, target=target, average='weighted')
+            else:
+                return scorer(ovr_results)
+
+    return thunked
+
+
+class ScoreTable(OWComponent, QObject):
+    shown_scores = \
+        Setting(set(chain(*BUILTIN_SCORERS_ORDER.values())))
+
+    shownScoresChanged = Signal()
+
+    class ItemDelegate(QStyledItemDelegate):
+        def sizeHint(self, *args):
+            size = super().sizeHint(*args)
+            return QSize(size.width(), size.height() + 6)
+
+    def __init__(self, master):
+        QObject.__init__(self)
+        OWComponent.__init__(self, master)
+
+        self.view = gui.TableView(
+            wordWrap=True, editTriggers=gui.TableView.NoEditTriggers
+        )
+        header = self.view.horizontalHeader()
+        header.setSectionResizeMode(QHeaderView.ResizeToContents)
+        header.setDefaultAlignment(Qt.AlignCenter)
+        header.setStretchLastSection(False)
+        header.setContextMenuPolicy(Qt.CustomContextMenu)
+        header.customContextMenuRequested.connect(self.show_column_chooser)
+
+        self.model = QStandardItemModel(master)
+        self.model.setHorizontalHeaderLabels(["Method"])
+        self.view.setModel(self.model)
+        self.view.setItemDelegate(self.ItemDelegate())
+
+    def _column_names(self):
+        return (self.model.horizontalHeaderItem(section).data(Qt.DisplayRole)
+                for section in range(1, self.model.columnCount()))
+
+    def show_column_chooser(self, pos):
+        # pylint doesn't know that self.shown_scores is a set, not a Setting
+        # pylint: disable=unsupported-membership-test
+        def update(col_name, checked):
+            if checked:
+                self.shown_scores.add(col_name)
+            else:
+                self.shown_scores.remove(col_name)
+            self._update_shown_columns()
+
+        menu = QMenu()
+        header = self.view.horizontalHeader()
+        for col_name in self._column_names():
+            action = menu.addAction(col_name)
+            action.setCheckable(True)
+            action.setChecked(col_name in self.shown_scores)
+            action.triggered.connect(partial(update, col_name))
+        menu.exec(header.mapToGlobal(pos))
+
+    def _update_shown_columns(self):
+        # pylint doesn't know that self.shown_scores is a set, not a Setting
+        # pylint: disable=unsupported-membership-test
+        header = self.view.horizontalHeader()
+        for section, col_name in enumerate(self._column_names(), start=1):
+            header.setSectionHidden(section, col_name not in self.shown_scores)
+        self.view.resizeColumnsToContents()
+        self.shownScoresChanged.emit()
+
+    def update_header(self, scorers):
+        # Set the correct horizontal header labels on the results_model.
+        self.model.setColumnCount(1 + len(scorers))
+        self.model.setHorizontalHeaderItem(0, QStandardItem("Model"))
+        for col, score in enumerate(scorers, start=1):
+            item = QStandardItem(score.name)
+            item.setToolTip(score.long_name)
+            self.model.setHorizontalHeaderItem(col, item)
+        self._update_shown_columns()


Test patch:
diff --git a/Orange/widgets/evaluate/tests/test_owtestlearners.py b/Orange/widgets/evaluate/tests/test_owtestlearners.py
--- a/Orange/widgets/evaluate/tests/test_owtestlearners.py
+++ b/Orange/widgets/evaluate/tests/test_owtestlearners.py
@@ -1,11 +1,9 @@
 # pylint: disable=missing-docstring
 # pylint: disable=protected-access
-import collections
 import unittest
 
 import numpy as np
-from AnyQt.QtWidgets import QMenu
-from AnyQt.QtCore import QPoint, Qt
+from AnyQt.QtCore import Qt
 from AnyQt.QtTest import QTest
 
 from Orange.classification import MajorityLearner, LogisticRegressionLearner
@@ -18,6 +16,7 @@
 from Orange.regression import MeanLearner
 from Orange.widgets.evaluate.owtestlearners import (
     OWTestLearners, results_one_vs_rest)
+from Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER
 from Orange.widgets.settings import (
     ClassValuesContextHandler, PerfectDomainContextHandler)
 from Orange.widgets.tests.base import WidgetTest
@@ -120,63 +119,6 @@ def test_CrossValidationByFeature(self):
         self.assertEqual(self.widget.resampling, OWTestLearners.KFold)
         self.assertFalse(self.widget.features_combo.isEnabled())
 
-    def test_update_shown_columns(self):
-        w = self.widget  #: OWTestLearners
-        all, shown = "MABDEFG", "ABDF"
-        header = w.view.horizontalHeader()
-        w.shown_scores = set(shown)
-        w.result_model.setHorizontalHeaderLabels(list(all))
-        w._update_shown_columns()
-        for i, name in enumerate(all):
-            self.assertEqual(name == "M" or name in shown,
-                             not header.isSectionHidden(i),
-                             msg="error in section {}({})".format(i, name))
-
-        w.shown_scores = set()
-        w._update_shown_columns()
-        for i, name in enumerate(all):
-            self.assertEqual(i == 0,
-                             not header.isSectionHidden(i),
-                             msg="error in section {}({})".format(i, name))
-
-    def test_show_column_chooser(self):
-        w = self.widget  #: OWTestLearners
-        all, shown = "MABDEFG", "ABDF"
-        header = w.view.horizontalHeader()
-        w.shown_scores = set(shown)
-        w.result_model.setHorizontalHeaderLabels(list(all))
-        w._update_shown_columns()
-
-        actions = collections.OrderedDict()
-        menu_add_action = QMenu.addAction
-
-        def addAction(menu, a):
-            action = menu_add_action(menu, a)
-            actions[a] = action
-            return action
-
-        def execmenu(*_):
-            self.assertEqual(list(actions), list(all)[1:])
-            for name, action in actions.items():
-                self.assertEqual(action.isChecked(), name in shown)
-            actions["E"].triggered.emit(True)
-            self.assertEqual(w.shown_scores, set("ABDEF"))
-            actions["B"].triggered.emit(False)
-            self.assertEqual(w.shown_scores, set("ADEF"))
-            for i, name in enumerate(all):
-                self.assertEqual(name == "M" or name in "ADEF",
-                                 not header.isSectionHidden(i),
-                                 msg="error in section {}({})".format(i, name))
-
-        # We must patch `QMenu.exec` because the Qt would otherwise (invisibly)
-        # show the popup and wait for the user.
-        # Assertions are made within `menuexec` since they check the
-        # instances of `QAction`, which are invalid (destroyed by Qt?) after
-        # `menuexec` finishes.
-        with unittest.mock.patch("AnyQt.QtWidgets.QMenu.addAction", addAction),\
-                unittest.mock.patch("AnyQt.QtWidgets.QMenu.exec", execmenu):
-            w.show_column_chooser(QPoint(0, 0))
-
     def test_migrate_removes_invalid_contexts(self):
         context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])
         context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)
@@ -246,7 +188,7 @@ class NewClassificationScore(ClassificationScore):
             class NewRegressionScore(RegressionScore):
                 pass
 
-            builtins = self.widget.BUILTIN_ORDER
+            builtins = BUILTIN_SCORERS_ORDER
             self.send_signal("Data", Table("iris"))
             scorer_names = [scorer.name for scorer in self.widget.scorers]
             self.assertEqual(
@@ -275,22 +217,23 @@ class NewRegressionScore(RegressionScore):
     def test_target_changing(self):
         data = Table("iris")
         w = self.widget  #: OWTestLearners
+        model = w.score_table.model
 
         w.n_folds = 2
         self.send_signal(self.widget.Inputs.train_data, data)
         self.send_signal(self.widget.Inputs.learner,
                          LogisticRegressionLearner(), 0, wait=5000)
 
-        average_auc = float(w.view.model().item(0, 1).text())
+        average_auc = float(model.item(0, 1).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-setosa")
-        setosa_auc = float(w.view.model().item(0, 1).text())
+        setosa_auc = float(model.item(0, 1).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-versicolor")
-        versicolor_auc = float(w.view.model().item(0, 1).text())
+        versicolor_auc = float(model.item(0, 1).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-virginica")
-        virginica_auc = float(w.view.model().item(0, 1).text())
+        virginica_auc = float(model.item(0, 1).text())
 
         self.assertGreater(average_auc, versicolor_auc)
         self.assertGreater(average_auc, virginica_auc)
@@ -326,31 +269,29 @@ def __call__(self, data):
         self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)
 
         self.widget.show()
-        header = self.widget.view.horizontalHeader()
+        view = self.widget.score_table.view
+        header = view.horizontalHeader()
         QTest.mouseClick(header.viewport(), Qt.LeftButton)
 
         # Ensure that the click on header caused an ascending sort
         # Ascending sort means that wrong model should be listed first
         self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)
-        self.assertEqual(
-            self.widget.view.model().item(0, 0).text(),
-            "VersicolorLearner")
+        self.assertEqual(view.model().item(0, 0).text(), "VersicolorLearner")
 
         self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)
-        self.assertEqual(
-            self.widget.view.model().item(0, 0).text(),
-            "SetosaLearner")
+        self.assertEqual(view.model().item(0, 0).text(), "SetosaLearner")
 
         self.widget.hide()
 
     def _retrieve_scores(self):
         w = self.widget
-        auc = w.view.model().item(0, 1).text()
+        model = w.score_table.model
+        auc = model.item(0, 1).text()
         auc = float(auc) if auc != "" else None
-        ca = float(w.view.model().item(0, 2).text())
-        f1 = float(w.view.model().item(0, 3).text())
-        precision = float(w.view.model().item(0, 4).text())
-        recall = float(w.view.model().item(0, 5).text())
+        ca = float(model.item(0, 2).text())
+        f1 = float(model.item(0, 3).text())
+        precision = float(model.item(0, 4).text())
+        recall = float(model.item(0, 5).text())
         return auc, ca, f1, precision, recall
 
     def _test_scores(self, train_data, test_data, learner, sampling, n_folds):
diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py
new file mode 100644
--- /dev/null
+++ b/Orange/widgets/evaluate/tests/test_utils.py
@@ -0,0 +1,74 @@
+# pylint: disable=protected-access
+
+import unittest
+import collections
+
+from AnyQt.QtWidgets import QMenu
+from AnyQt.QtCore import QPoint
+
+from Orange.widgets.evaluate.utils import ScoreTable
+from Orange.widgets.tests.base import GuiTest
+
+
+class TestScoreTable(GuiTest):
+    def test_show_column_chooser(self):
+        score_table = ScoreTable(None)
+        view = score_table.view
+        all, shown = "MABDEFG", "ABDF"
+        header = view.horizontalHeader()
+        score_table.shown_scores = set(shown)
+        score_table.model.setHorizontalHeaderLabels(list(all))
+        score_table._update_shown_columns()
+
+        actions = collections.OrderedDict()
+        menu_add_action = QMenu.addAction
+
+        def addAction(menu, a):
+            action = menu_add_action(menu, a)
+            actions[a] = action
+            return action
+
+        def execmenu(*_):
+            self.assertEqual(list(actions), list(all)[1:])
+            for name, action in actions.items():
+                self.assertEqual(action.isChecked(), name in shown)
+            actions["E"].triggered.emit(True)
+            self.assertEqual(score_table.shown_scores, set("ABDEF"))
+            actions["B"].triggered.emit(False)
+            self.assertEqual(score_table.shown_scores, set("ADEF"))
+            for i, name in enumerate(all):
+                self.assertEqual(name == "M" or name in "ADEF",
+                                 not header.isSectionHidden(i),
+                                 msg="error in section {}({})".format(i, name))
+
+        # We must patch `QMenu.exec` because the Qt would otherwise (invisibly)
+        # show the popup and wait for the user.
+        # Assertions are made within `menuexec` since they check the
+        # instances of `QAction`, which are invalid (destroyed by Qt?) after
+        # `menuexec` finishes.
+        with unittest.mock.patch("AnyQt.QtWidgets.QMenu.addAction", addAction), \
+             unittest.mock.patch("AnyQt.QtWidgets.QMenu.exec", execmenu):
+            score_table.show_column_chooser(QPoint(0, 0))
+
+    def test_update_shown_columns(self):
+        score_table = ScoreTable(None)
+        view = score_table.view
+        all, shown = "MABDEFG", "ABDF"
+        header = view.horizontalHeader()
+        score_table.shown_scores = set(shown)
+        score_table.model.setHorizontalHeaderLabels(list(all))
+        score_table._update_shown_columns()
+        for i, name in enumerate(all):
+            self.assertEqual(name == "M" or name in shown,
+                             not header.isSectionHidden(i),
+                             msg="error in section {}({})".format(i, name))
+
+        score_table.shown_scores = set()
+        score_table._update_shown_columns()
+        for i, name in enumerate(all):
+            self.assertEqual(i == 0,
+                             not header.isSectionHidden(i),
+                             msg="error in section {}({})".format(i, name))
+
+if __name__ == "__main__":
+    unittest.main()