Repository: biolab/orange3

Issue:
Test & Score: bring back k < class members Warning
<!-- 
Thanks for taking the time to report a bug!
If you're raising an issue about an add-on (i.e., installed via Options > Add-ons), raise an issue in the relevant add-on's issue tracker instead. See: https://github.com/biolab?q=orange3
To fix the bug, we need to be able to reproduce it. Please answer the following questions to the best of your ability.
-->


- [ ] **What's wrong?**
<!-- Be specific, clear, and concise. Include screenshots if relevant. -->
<!-- If you're getting an error message, copy it, and enclose it with three backticks (```). -->
Test&Score used to warn when k was less than the instances in least populated class. Now the Warning is gone.




- [ ] **How can we reproduce the problem?** 
<!-- Upload a zip with the .ows file and data. -->
<!-- Describe the steps (open this widget, click there, then add this...) -->
File (zoo) - Log Reg + Test and Score (k=5).




- [ ] **Which version of Orange are you using? How did you install Orange?**
<!-- See "Help → About → Version" or `Orange.version.full_version` in code -->
3.27.dev




- [ ] **What is your operating system?**
OSX







Discussion:

Resolving patch:
diff --git a/Orange/widgets/evaluate/owtestandscore.py b/Orange/widgets/evaluate/owtestandscore.py
--- a/Orange/widgets/evaluate/owtestandscore.py
+++ b/Orange/widgets/evaluate/owtestandscore.py
@@ -207,12 +207,16 @@ class Warning(OWWidget.Warning):
         scores_not_computed = Msg("Some scores could not be computed.")
         test_data_unused = Msg("Test data is present but unused. "
                                "Select 'Test on test data' to use it.")
+        cant_stratify = \
+            Msg("Can't run stratified {}-fold cross validation; "
+                "the least common class has only {} instances.")
 
     class Information(OWWidget.Information):
         data_sampled = Msg("Train data has been sampled")
         test_data_sampled = Msg("Test data has been sampled")
         test_data_transformed = Msg(
             "Test data has been transformed to match the train data.")
+        cant_stratify_numeric = Msg("Stratification is ignored for regression")
 
     def __init__(self):
         super().__init__()
@@ -921,6 +925,8 @@ def __update(self):
         self.Warning.test_data_unused.clear()
         self.Error.test_data_incompatible.clear()
         self.Warning.test_data_missing.clear()
+        self.Warning.cant_stratify.clear()
+        self.Information.cant_stratify_numeric.clear()
         self.Information.test_data_transformed(
             shown=self.resampling == self.TestOnTest
             and self.data is not None
@@ -931,7 +937,7 @@ def __update(self):
         self.Error.too_many_folds.clear()
         self.error()
 
-        # check preconditions and return early
+        # check preconditions and return early or show warnings
         if self.data is None:
             self.__state = State.Waiting
             self.commit()
@@ -940,12 +946,24 @@ def __update(self):
             self.__state = State.Waiting
             self.commit()
             return
-        if self.resampling == OWTestAndScore.KFold and \
-                len(self.data) < self.NFolds[self.n_folds]:
-            self.Error.too_many_folds()
-            self.__state = State.Waiting
-            self.commit()
-            return
+        if self.resampling == OWTestAndScore.KFold:
+            k = self.NFolds[self.n_folds]
+            if len(self.data) < k:
+                self.Error.too_many_folds()
+                self.__state = State.Waiting
+                self.commit()
+                return
+            do_stratify = self.cv_stratified
+            if do_stratify:
+                if self.data.domain.class_var.is_discrete:
+                    least = min(filter(None,
+                                       np.bincount(self.data.Y.astype(int))))
+                    if least < k:
+                        self.Warning.cant_stratify(k, least)
+                        do_stratify = False
+                else:
+                    self.Information.cant_stratify_numeric()
+                    do_stratify = False
 
         elif self.resampling == OWTestAndScore.TestOnTest:
             if self.test_data is None:
@@ -985,7 +1003,8 @@ def __update(self):
             if self.resampling == OWTestAndScore.KFold:
                 sampler = Orange.evaluation.CrossValidation(
                     k=self.NFolds[self.n_folds],
-                    random_state=rstate)
+                    random_state=rstate,
+                    stratified=do_stratify)
             elif self.resampling == OWTestAndScore.FeatureFold:
                 sampler = Orange.evaluation.CrossValidationFeature(
                     feature=self.fold_feature)


Test patch:
diff --git a/Orange/widgets/evaluate/tests/test_owtestandscore.py b/Orange/widgets/evaluate/tests/test_owtestandscore.py
--- a/Orange/widgets/evaluate/tests/test_owtestandscore.py
+++ b/Orange/widgets/evaluate/tests/test_owtestandscore.py
@@ -422,6 +422,52 @@ def test_scores_cross_validation(self):
                     OWTestAndScore.KFold, 0),
                 (0.8, 0.5, 0.5, 0.5, 0.5))))
 
+    def test_no_stratification(self):
+        w = self.widget
+        w.cv_stratified = True
+        self._test_scores(
+            Table("zoo"), None, MajorityLearner(),
+            OWTestAndScore.KFold, 2)
+        self.assertTrue(w.Warning.cant_stratify.is_shown())
+
+        w.controls.cv_stratified.click()
+        self.assertFalse(w.Warning.cant_stratify.is_shown())
+
+        w.controls.cv_stratified.click()
+        self.assertTrue(w.Warning.cant_stratify.is_shown())
+
+        w.controls.n_folds.setCurrentIndex(0)
+        w.controls.n_folds.activated[int].emit(0)
+        self.assertFalse(w.Warning.cant_stratify.is_shown())
+
+        w.controls.n_folds.setCurrentIndex(2)
+        w.controls.n_folds.activated[int].emit(2)
+        self.assertTrue(w.Warning.cant_stratify.is_shown())
+
+        self._test_scores(
+            Table("iris"), None, MajorityLearner(), OWTestAndScore.KFold, 2)
+        self.assertFalse(w.Warning.cant_stratify.is_shown())
+
+        self._test_scores(
+            Table("zoo"), None, MajorityLearner(), OWTestAndScore.KFold, 2)
+        self.assertTrue(w.Warning.cant_stratify.is_shown())
+
+        self._test_scores(
+            Table("housing"), None, MeanLearner(), OWTestAndScore.KFold, 2)
+        self.assertFalse(w.Warning.cant_stratify.is_shown())
+        self.assertTrue(w.Information.cant_stratify_numeric.is_shown())
+
+        w.controls.cv_stratified.click()
+        self.assertFalse(w.Warning.cant_stratify.is_shown())
+
+    def test_too_many_folds(self):
+        w = self.widget
+        w.controls.resampling.buttons[OWTestAndScore.KFold].click()
+        w.n_folds = 3
+        self.send_signal(w.Inputs.train_data, Table("zoo")[:8])
+        self.send_signal(w.Inputs.learner, MajorityLearner(), 0, wait=5000)
+        self.assertTrue(w.Error.too_many_folds.is_shown())
+
     def test_no_pregressbar_warning(self):
         data = Table("iris")[::15]