Repository: biolab/orange3

Issue:
Scorers and instances with unknown outcomes
**What's wrong?**

When scorers, e.g. in Predictions, encounter data instances with unknown classes, they fail; the widget shows an error in the status bar, saying that some scorers have failed.

This is OK, but I wonder whether scorers could ignore such instances, or, perhaps better, the widget would remove them and show a warning instead of an error.

**How can we reproduce the problem?** 

Train a tree from on this data (select data instances with known class value) and make predictions (using Predictions) for all data.

[Mushrooms.xlsx](https://github.com/biolab/orange3/files/7881889/Mushrooms.xlsx)


Discussion:

Resolving patch:
diff --git a/Orange/widgets/evaluate/owpredictions.py b/Orange/widgets/evaluate/owpredictions.py
--- a/Orange/widgets/evaluate/owpredictions.py
+++ b/Orange/widgets/evaluate/owpredictions.py
@@ -65,6 +65,8 @@ class Warning(OWWidget.Warning):
         empty_data = Msg("Empty dataset")
         wrong_targets = Msg(
             "Some model(s) predict a different target (see more ...)\n{}")
+        missing_targets = Msg("Instances with missing targets "
+                              "are ignored while scoring.")
 
     class Error(OWWidget.Error):
         predictor_failed = Msg("Some predictor(s) failed (see more ...)\n{}")
@@ -291,18 +293,38 @@ def _update_scores(self):
                 continue
             row = [QStandardItem(learner_name(pred.predictor)),
                    QStandardItem("N/A"), QStandardItem("N/A")]
-            for scorer in scorers:
-                item = QStandardItem()
-                try:
-                    score = scorer_caller(scorer, results)()[0]
-                    item.setText(f"{score:.3f}")
-                except Exception as exc:  # pylint: disable=broad-except
-                    item.setToolTip(str(exc))
-                    # false pos.; pylint: disable=unsupported-membership-test
-                    if scorer.name in self.score_table.shown_scores:
-                        errors.append(str(exc))
-                row.append(item)
-            self.score_table.model.appendRow(row)
+
+            try:
+                actual = results.actual
+                predicted = results.predicted
+                probabilities = results.probabilities
+
+                mask = numpy.isnan(results.actual)
+                no_targets = mask.sum() == len(results.actual)
+                results.actual = results.actual[~mask]
+                results.predicted = results.predicted[:, ~mask]
+                results.probabilities = results.probabilities[:, ~mask]
+
+                for scorer in scorers:
+                    item = QStandardItem()
+                    if no_targets:
+                        item.setText("NA")
+                    else:
+                        try:
+                            score = scorer_caller(scorer, results)()[0]
+                            item.setText(f"{score:.3f}")
+                        except Exception as exc:  # pylint: disable=broad-except
+                            item.setToolTip(str(exc))
+                            # false pos.; pylint: disable=unsupported-membership-test
+                            if scorer.name in self.score_table.shown_scores:
+                                errors.append(str(exc))
+                    row.append(item)
+                self.score_table.model.appendRow(row)
+
+            finally:
+                results.actual = actual
+                results.predicted = predicted
+                results.probabilities = probabilities
 
         view = self.score_table.view
         if model.rowCount():
@@ -332,8 +354,12 @@ def _set_errors(self):
                              if isinstance(p.results, Results)
                              and p.results.probabilities is None))
             self.Warning.wrong_targets(inv_targets, shown=bool(inv_targets))
+
+            show_warning = numpy.isnan(self.data.Y).any() and self.predictors
+            self.Warning.missing_targets(shown=show_warning)
         else:
             self.Warning.wrong_targets.clear()
+            self.Warning.missing_targets.clear()
 
     def _get_details(self):
         details = "Data:<br>"


Test patch:
diff --git a/Orange/widgets/evaluate/tests/test_owpredictions.py b/Orange/widgets/evaluate/tests/test_owpredictions.py
--- a/Orange/widgets/evaluate/tests/test_owpredictions.py
+++ b/Orange/widgets/evaluate/tests/test_owpredictions.py
@@ -11,6 +11,7 @@
 from Orange.base import Model
 from Orange.classification import LogisticRegressionLearner
 from Orange.data.io import TabReader
+from Orange.regression import LinearRegressionLearner
 from Orange.widgets.tests.base import WidgetTest
 from Orange.widgets.evaluate.owpredictions import (
     OWPredictions, SharedSelectionModel, SharedSelectionStore, DataModel,
@@ -33,6 +34,7 @@ class TestOWPredictions(WidgetTest):
     def setUp(self):
         self.widget = self.create_widget(OWPredictions)  # type: OWPredictions
         self.iris = Table("iris")
+        self.housing = Table("housing")
 
     def test_rowCount_from_model(self):
         """Don't crash if the bottom row is visible"""
@@ -519,6 +521,62 @@ def check_evres(expected):
         self.send_signal(w.Inputs.predictors, p2, 2)
         check_evres(["P1", "P3", "P2"])
 
+    def test_missing_target_cls(self):
+        mask = np.zeros(len(self.iris), dtype=bool)
+        mask[::2] = True
+        train_data = self.iris[~mask]
+        predict_data = self.iris[mask]
+        model = LogisticRegressionLearner()(train_data)
+
+        self.send_signal(self.widget.Inputs.predictors, model)
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertFalse(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        with predict_data.unlocked():
+            predict_data.Y[0] = np.nan
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertTrue(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        with predict_data.unlocked():
+            predict_data.Y[:] = np.nan
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertTrue(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        self.send_signal(self.widget.Inputs.predictors, None)
+        self.assertFalse(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+    def test_missing_target_reg(self):
+        mask = np.zeros(len(self.housing), dtype=bool)
+        mask[::2] = True
+        train_data = self.housing[~mask]
+        predict_data = self.housing[mask]
+        model = LinearRegressionLearner()(train_data)
+
+        self.send_signal(self.widget.Inputs.predictors, model)
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertFalse(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        with predict_data.unlocked():
+            predict_data.Y[0] = np.nan
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertTrue(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        with predict_data.unlocked():
+            predict_data.Y[:] = np.nan
+        self.send_signal(self.widget.Inputs.data, predict_data)
+        self.assertTrue(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
+        self.send_signal(self.widget.Inputs.predictors, None)
+        self.assertFalse(self.widget.Warning.missing_targets.is_shown())
+        self.assertFalse(self.widget.Error.scorer_failed.is_shown())
+
 
 class SelectionModelTest(unittest.TestCase):
     def setUp(self):