Repository: biolab/orange3

Issue:
Predictions should have output with annotated data
**What's your use case?**

1. Train a model, test it in Predictions.
2. In Predictions, select all rows for which the predicted probability of the positive outcome is above certain threshold.
3. Feed *the entire data set* into Distributions (or another widget) to analyze predicted vs. actual class distributions by plotting the outcome and grouping by selection (or vice versa, depending on the intent of visualization).

One can't do the last step because Prediction's only output is the selected data. 

**What's your proposed solution?**

Predictions is similar to Table, so it could have a similar output with annotated data. The current output would be renamed to Selection.

**Are there any alternative solutions?**

One can connect Predictions to Table and select rows there. This doesn't look so elegant (Table is not formatted specifically for predictions) and involves an extra step that could be avoided without any cost to gui or to implementation of the Predictions widget - especially if the output signal is lazy.


Discussion:

Resolving patch:
diff --git a/Orange/widgets/evaluate/owpredictions.py b/Orange/widgets/evaluate/owpredictions.py
--- a/Orange/widgets/evaluate/owpredictions.py
+++ b/Orange/widgets/evaluate/owpredictions.py
@@ -18,6 +18,7 @@
 
 from orangecanvas.utils.localization import pl
 from orangewidget.utils.itemmodels import AbstractSortTableModel
+from orangewidget.utils.signals import LazyValue
 
 import Orange
 from Orange.evaluation import Results
@@ -31,6 +32,8 @@
 from Orange.widgets.utils.widgetpreview import WidgetPreview
 from Orange.widgets.widget import OWWidget, Msg, Input, Output, MultiInput
 from Orange.widgets.utils.itemmodels import TableModel
+from Orange.widgets.utils.annotated_data import lazy_annotated_table, \
+    domain_with_annotation_column, create_annotated_table
 from Orange.widgets.utils.sql import check_sql_input
 from Orange.widgets.utils.state_summary import format_summary_details
 from Orange.widgets.utils.colorpalettes import LimitedDiscretePalette
@@ -72,7 +75,9 @@ class Inputs:
         predictors = MultiInput("Predictors", Model, filter_none=True)
 
     class Outputs:
-        predictions = Output("Predictions", Orange.data.Table)
+        selected_predictions = Output("Selected Predictions", Orange.data.Table,
+                                      default=True, replaces=["Predictions"])
+        annotated = Output("Predictions", Orange.data.Table)
         evaluation_results = Output("Evaluation Results", Results)
 
     class Warning(OWWidget.Warning):
@@ -814,7 +819,8 @@ def _commit_evaluation_results(self):
 
     def _commit_predictions(self):
         if not self.data:
-            self.Outputs.predictions.send(None)
+            self.Outputs.selected_predictions.send(None)
+            self.Outputs.annotated.send(None)
             return
 
         newmetas = []
@@ -850,17 +856,26 @@ def _commit_predictions(self):
         predmodel = self.predictionsview.model()
         assert datamodel is not None  # because we have data
         assert self.selection_store is not None
-        rows = numpy.array(list(self.selection_store.rows))
+        rows = numpy.array(list(self.selection_store.rows), dtype=int)
         if rows.size:
+            domain, _ = domain_with_annotation_column(predictions)
+            annotated_data = LazyValue[Orange.data.Table](
+                lambda: create_annotated_table(
+                    predictions, rows)[datamodel.mapToSourceRows(...)],
+                length=len(predictions), domain=domain)
+
             # Reorder rows as they are ordered in view
             shown_rows = datamodel.mapFromSourceRows(rows)
             rows = rows[numpy.argsort(shown_rows)]
-            predictions = predictions[rows]
-        elif datamodel.sortColumn() >= 0 \
-                or predmodel is not None and predmodel.sortColumn() > 0:
-            # No selection: output all, but in the shown order
-            predictions = predictions[datamodel.mapToSourceRows(...)]
-        self.Outputs.predictions.send(predictions)
+            selected = predictions[rows]
+        else:
+            if datamodel.sortColumn() >= 0 \
+                    or predmodel is not None and predmodel.sortColumn() > 0:
+                predictions = predictions[datamodel.mapToSourceRows(...)]
+            selected = predictions
+            annotated_data = lazy_annotated_table(predictions, rows)
+        self.Outputs.selected_predictions.send(selected)
+        self.Outputs.annotated.send(annotated_data)
 
     def _add_classification_out_columns(self, slot, newmetas, newcolumns, index):
         pred = slot.predictor


Test patch:
diff --git a/Orange/widgets/evaluate/tests/test_owpredictions.py b/Orange/widgets/evaluate/tests/test_owpredictions.py
--- a/Orange/widgets/evaluate/tests/test_owpredictions.py
+++ b/Orange/widgets/evaluate/tests/test_owpredictions.py
@@ -36,6 +36,7 @@
 from Orange.evaluation import Results
 from Orange.widgets.tests.utils import excepthook_catch, \
     possible_duplicate_table, simulate
+from Orange.widgets.utils.annotated_data import ANNOTATED_DATA_FEATURE_NAME
 from Orange.widgets.utils.colorpalettes import LimitedDiscretePalette
 
 
@@ -62,7 +63,7 @@ def test_nan_target_input(self):
         yvec = data.get_column(data.domain.class_var)
         self.send_signal(self.widget.Inputs.data, data)
         self.send_signal(self.widget.Inputs.predictors, ConstantLearner()(data), 1)
-        pred = self.get_output(self.widget.Outputs.predictions)
+        pred = self.get_output(self.widget.Outputs.selected_predictions)
         self.assertIsInstance(pred, Table)
         np.testing.assert_array_equal(
             yvec, pred.get_column(data.domain.class_var))
@@ -92,7 +93,7 @@ def test_no_values_target(self):
         test = Table(domain, np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]]),
                      np.full((3, 1), np.nan))
         self.send_signal(self.widget.Inputs.data, test)
-        pred = self.get_output(self.widget.Outputs.predictions)
+        pred = self.get_output(self.widget.Outputs.selected_predictions)
         self.assertEqual(len(pred), len(test))
 
         results = self.get_output(self.widget.Outputs.evaluation_results)
@@ -145,7 +146,7 @@ def test_no_class_on_test(self):
         no_class = titanic.transform(Domain(titanic.domain.attributes, None))
         self.send_signal(self.widget.Inputs.predictors, majority_titanic, 1)
         self.send_signal(self.widget.Inputs.data, no_class)
-        out = self.get_output(self.widget.Outputs.predictions)
+        out = self.get_output(self.widget.Outputs.selected_predictions)
         np.testing.assert_allclose(out.get_column("constant"), 0)
 
         predmodel = self.widget.predictionsview.model()
@@ -500,7 +501,7 @@ def test_unique_output_domain(self):
         self.send_signal(self.widget.Inputs.data, data)
         self.send_signal(self.widget.Inputs.predictors, predictor)
 
-        output = self.get_output(self.widget.Outputs.predictions)
+        output = self.get_output(self.widget.Outputs.selected_predictions)
         self.assertEqual(output.domain.metas[0].name, 'constant (1)')
 
     def test_select(self):
@@ -515,6 +516,92 @@ def test_select(self):
                for index in self.widget.dataview.selectionModel().selectedIndexes()}
         self.assertEqual(sel, {(1, col) for col in range(5)})
 
+    def test_selection_output(self):
+        log_reg_iris = LogisticRegressionLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, log_reg_iris)
+        self.send_signal(self.widget.Inputs.data, self.iris)
+
+        selmodel = self.widget.dataview.selectionModel()
+        pred_model = self.widget.predictionsview.model()
+
+        selmodel.select(self.widget.dataview.model().index(1, 0), QItemSelectionModel.Select)
+        selmodel.select(self.widget.dataview.model().index(3, 0), QItemSelectionModel.Select)
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), 2)
+        self.assertEqual(output[0], self.iris[1])
+        self.assertEqual(output[1], self.iris[3])
+        output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertEqual(np.sum(col), 2)
+        self.assertEqual(col[1], 1)
+        self.assertEqual(col[3], 1)
+
+        pred_model.sort(0)
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), 2)
+        self.assertEqual(output[0], self.iris[1])
+        self.assertEqual(output[1], self.iris[3])
+        ann_output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(ann_output), len(self.iris))
+        col = ann_output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertEqual(np.sum(col), 2)
+        np.testing.assert_array_equal(ann_output[col == 1].X, output.X)
+
+        pred_model.sort(0, Qt.DescendingOrder)
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), 2)
+        self.assertEqual(output[0], self.iris[3])
+        self.assertEqual(output[1], self.iris[1])
+        ann_output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(ann_output), len(self.iris))
+        col = ann_output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertEqual(np.sum(col), 2)
+        np.testing.assert_array_equal(ann_output[col == 1].X, output.X)
+
+    def test_no_selection_output(self):
+        log_reg_iris = LogisticRegressionLearner()(self.iris)
+        self.send_signal(self.widget.Inputs.predictors, log_reg_iris)
+        self.send_signal(self.widget.Inputs.data, self.iris)
+
+        data_model = self.widget.dataview.model()
+
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), len(self.iris))
+        output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertFalse(np.any(col))
+
+        data_model.sort(2)
+        col_name = data_model.headerData(2, Qt.Horizontal, Qt.DisplayRole)  # "sepal width"
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(col_name)
+        self.assertTrue(np.all(col[1:] >= col[:-1]))
+
+        output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(col_name)
+        self.assertTrue(np.all(col[1:] >= col[:-1]))
+        col = output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertFalse(np.any(col))
+
+        data_model.sort(2, Qt.DescendingOrder)
+        col_name = data_model.headerData(2, Qt.Horizontal, Qt.DisplayRole)  # "sepal width"
+        output = self.get_output(self.widget.Outputs.selected_predictions)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(col_name)
+        self.assertTrue(np.all(col[1:] <= col[:-1]))
+
+        output = self.get_output(self.widget.Outputs.annotated)
+        self.assertEqual(len(output), len(self.iris))
+        col = output.get_column(col_name)
+        self.assertTrue(np.all(col[1:] <= col[:-1]))
+        col = output.get_column(ANNOTATED_DATA_FEATURE_NAME)
+        self.assertFalse(np.any(col))
+
+
     def test_select_data_first(self):
         log_reg_iris = LogisticRegressionLearner()(self.iris)
         self.send_signal(self.widget.Inputs.data, self.iris)
@@ -537,7 +624,7 @@ def test_selection_in_setting(self):
                for index in widget.dataview.selectionModel().selectedIndexes()}
         self.assertEqual(sel, {(row, col)
                                for row in [1, 3, 4] for col in range(5)})
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         exp = self.iris[np.array([1, 3, 4])]
         np.testing.assert_equal(out.X, exp.X)
 
@@ -883,32 +970,32 @@ def test_output_wrt_shown_probs_1(self):
 
         widget.shown_probs = widget.NO_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 1, 2])
 
         widget.shown_probs = widget.DATA_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 0, 110, 2, 210, 211])
 
         widget.shown_probs = widget.MODEL_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 110, 111, 2, 210, 211, 212])
 
         widget.shown_probs = widget.BOTH_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 110, 2, 210, 211])
 
         widget.shown_probs = widget.BOTH_PROBS + 1
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 1, 0, 2, 210])
 
         widget.shown_probs = widget.BOTH_PROBS + 2
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 11, 1, 110, 2, 211])
 
     def test_output_wrt_shown_probs_2(self):
@@ -933,37 +1020,37 @@ def test_output_wrt_shown_probs_2(self):
 
         widget.shown_probs = widget.NO_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 1])
 
         widget.shown_probs = widget.DATA_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 0, 1, 110, 111, 112])
 
         widget.shown_probs = widget.MODEL_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 110, 111, 112])
 
         widget.shown_probs = widget.BOTH_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 110, 111, 112])
 
         widget.shown_probs = widget.BOTH_PROBS + 1
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 1, 110])
 
         widget.shown_probs = widget.BOTH_PROBS + 2
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 11, 1, 111])
 
         widget.shown_probs = widget.BOTH_PROBS + 3
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 0, 1, 112])
 
     def test_output_regression(self):
@@ -973,7 +1060,7 @@ def test_output_regression(self):
                          LinearRegressionLearner()(self.housing), 0)
         self.send_signal(widget.Inputs.predictors,
                          MeanLearner()(self.housing), 1)
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         np.testing.assert_equal(
             out.metas[:, [0, 2]],
             np.hstack([pred.results.predicted.T for pred in widget.predictors]))
@@ -1001,12 +1088,12 @@ def test_classless(self):
 
         widget.shown_probs = widget.NO_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 1, 2])
 
         widget.shown_probs = widget.MODEL_PROBS
         widget._commit_predictions()
-        out = self.get_output(widget.Outputs.predictions)
+        out = self.get_output(widget.Outputs.selected_predictions)
         self.assertEqual(list(out.metas[0]), [0, 10, 11, 1, 110, 111, 2, 210, 211, 212])
 
     @patch("Orange.widgets.evaluate.owpredictions.usable_scorers",
@@ -1047,7 +1134,7 @@ def test_multi_target_input(self):
 
         self.send_signal(widget.Inputs.data, data)
         self.send_signal(widget.Inputs.predictors, mock_model, 1)
-        pred = self.get_output(widget.Outputs.predictions)
+        pred = self.get_output(widget.Outputs.selected_predictions)
         self.assertIsInstance(pred, Table)
 
     def test_error_controls_visibility(self):
@@ -1202,7 +1289,7 @@ def test_output_error_reg(self):
         self.send_signal(self.widget.Inputs.predictors, lin_reg(data), 0)
         self.send_signal(self.widget.Inputs.predictors,
                          LinearRegressionLearner(fit_intercept=False)(data), 1)
-        pred = self.get_output(self.widget.Outputs.predictions)
+        pred = self.get_output(self.widget.Outputs.selected_predictions)
 
         names = ["", " (error)"]
         names = [f"{n}{i}" for i in ("", " (1)") for n in names]
@@ -1220,7 +1307,7 @@ def test_output_error_cls(self):
         with data.unlocked(data.Y):
             data.Y[1] = np.nan
         self.send_signal(self.widget.Inputs.data, data)
-        pred = self.get_output(self.widget.Outputs.predictions)
+        pred = self.get_output(self.widget.Outputs.selected_predictions)
 
         names = [""] + [f" ({v})" for v in
                         list(data.domain.class_var.values) + ["error"]]