Repository: deepchecks/deepchecks

Issue:
[FEAT] the unit test for suite should also validate that no condition raised an error
**Is your feature request related to a problem? Please describe.**
the unit test for suite should also validate no condition raised an error

condition errors are registered as warnings and are not checked


Discussion:

Resolving patch:
diff --git a/deepchecks/core/checks.py b/deepchecks/core/checks.py
--- a/deepchecks/core/checks.py
+++ b/deepchecks/core/checks.py
@@ -68,7 +68,7 @@ def conditions_decision(self, result: CheckResult) -> List[ConditionResult]:
                 output = condition.function(result.value, **condition.params)
             except Exception as e:
                 msg = f'Exception in condition: {e.__class__.__name__}: {str(e)}'
-                output = ConditionResult(ConditionCategory.WARN, msg)
+                output = ConditionResult(ConditionCategory.ERROR, msg)
             if isinstance(output, bool):
                 output = ConditionResult(ConditionCategory.PASS if output else ConditionCategory.FAIL)
             elif not isinstance(output, ConditionResult):
diff --git a/deepchecks/core/condition.py b/deepchecks/core/condition.py
--- a/deepchecks/core/condition.py
+++ b/deepchecks/core/condition.py
@@ -11,6 +11,7 @@
 """Module containing all the base classes for checks."""
 import enum
 from typing import Callable, Dict, cast
+
 from deepchecks.core.errors import DeepchecksValueError
 
 
@@ -59,6 +60,7 @@ class ConditionCategory(enum.Enum):
     FAIL = 'FAIL'
     WARN = 'WARN'
     PASS = 'PASS'
+    ERROR = 'ERROR'
 
 
 class ConditionResult:
@@ -106,11 +108,12 @@ def priority(self) -> int:
             condition priority value.
         """
         if self.category == ConditionCategory.PASS:
-            return 3
+            return 4
         elif self.category == ConditionCategory.FAIL:
             return 1
-        else:
+        elif self.category == ConditionCategory.WARN:
             return 2
+        return 3  # if error
 
     @property
     def is_pass(self) -> bool:
@@ -123,8 +126,9 @@ def get_icon(self):
             return '<div style="color: green;text-align: center">\U00002713</div>'
         elif self.category == ConditionCategory.FAIL:
             return '<div style="color: red;text-align: center">\U00002716</div>'
-        else:
+        elif self.category == ConditionCategory.WARN:
             return '<div style="color: orange;text-align: center;font-weight:bold">\U00000021</div>'
+        return '<div style="color: firebrick;text-align: center;font-weight:bold">\U00002048</div>'
 
     def __repr__(self):
         """Return string representation for printing."""
diff --git a/deepchecks/core/display_suite.py b/deepchecks/core/display_suite.py
--- a/deepchecks/core/display_suite.py
+++ b/deepchecks/core/display_suite.py
@@ -296,7 +296,8 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
     icons = """
     <span style="color: green;display:inline-block">\U00002713</span> /
     <span style="color: red;display:inline-block">\U00002716</span> /
-    <span style="color: orange;font-weight:bold;display:inline-block">\U00000021</span>
+    <span style="color: orange;font-weight:bold;display:inline-block">\U00000021</span> /
+    <span style="color: firebrick;font-weight:bold;display:inline-block">\U00002048</span>
     """
 
     check_names = list(set(it.check.name() for it in results))
@@ -316,8 +317,8 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
         <h1 id="summary_{unique_id}">{suite_name}</h1>
         <p>
             {prologue}<br>
-            Each check may contain conditions (which will result in pass / fail / warning, represented by {icons})
-            as well as other outputs such as plots or tables.<br>
+            Each check may contain conditions (which will result in pass / fail / warning / error
+            , represented by {icons}) as well as other outputs such as plots or tables.<br>
             Suites, checks and conditions can all be modified. Read more about
             <a href={suite_creation_example_link} target="_blank">custom suites</a>.
         </p>


Test patch:
diff --git a/tests/base/check_test.py b/tests/base/check_test.py
--- a/tests/base/check_test.py
+++ b/tests/base/check_test.py
@@ -140,7 +140,7 @@ def raise_(ex):  # just to test error in condition
         all_of(
             has_property('name', 'condition F'),
             has_property('is_pass', equal_to(False)),
-            has_property('category', ConditionCategory.WARN),
+            has_property('category', ConditionCategory.ERROR),
             has_property('details', 'Exception in condition: Exception: fail')
         )
     ))
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,8 +11,10 @@
 """Represents fixtures for unit testing using pytest."""
 # Disable this pylint check since we use this convention in pytest fixtures
 #pylint: disable=redefined-outer-name
-from typing import Any, Tuple
+from typing import Any, Dict, Tuple, Optional
 
+from hamcrest import assert_that, instance_of, only_contains, any_of
+from hamcrest.core.matcher import Matcher
 import numpy as np
 import pytest
 import pandas as pd
@@ -27,12 +29,62 @@
 from xgboost import XGBClassifier, XGBRegressor
 from catboost import CatBoostClassifier, CatBoostRegressor
 from lightgbm import LGBMClassifier, LGBMRegressor
+from deepchecks.core.checks import SingleDatasetBaseCheck
+from deepchecks.core.condition import ConditionCategory
+from deepchecks.core.errors import DeepchecksBaseError
+from deepchecks.core.suite import BaseSuite, SuiteResult
 
 from deepchecks.tabular import Dataset, TrainTestCheck, Context
-from deepchecks.core.check_result import CheckResult
+from deepchecks.core.check_result import CheckFailure, CheckResult
 
 from .vision.vision_conftest import * #pylint: disable=wildcard-import, unused-wildcard-import
 
+def get_expected_results_length(suite: BaseSuite, args: Dict):
+    num_single = len([c for c in suite.checks.values() if isinstance(c, SingleDatasetBaseCheck)])
+    num_others = len(suite.checks.values()) - num_single
+    multiply = 0
+    if 'train_dataset' in args:
+        multiply += 1
+    if 'test_dataset' in args:
+        multiply += 1
+    # If no train and no test (only model) there will be single result of check failure
+    if multiply == 0:
+        multiply = 1
+
+    return num_single * multiply + num_others
+
+def validate_suite_result(
+    result: SuiteResult,
+    min_length: int,
+    exception_matcher: Optional[Matcher] = None
+):
+    assert_that(result, instance_of(SuiteResult))
+    assert_that(result.results, instance_of(list))
+    assert_that(len(result.results) >= min_length)
+
+    exception_matcher = exception_matcher or only_contains(instance_of(DeepchecksBaseError))
+
+    assert_that(result.results, only_contains(any_of( # type: ignore
+        instance_of(CheckFailure),
+        instance_of(CheckResult),
+    )))
+
+    failures = [
+        it.exception
+        for it in result.results
+        if isinstance(it, CheckFailure)
+    ]
+
+    if len(failures) != 0:
+        assert_that(actual=failures, matcher=exception_matcher) # type: ignore
+
+    for check_result in result.results:
+        if isinstance(check_result, CheckResult) and check_result.have_conditions():
+            for cond in check_result.conditions_results:
+                assert_that(cond.category, any_of(ConditionCategory.PASS,
+                                                  ConditionCategory.WARN,
+                                                  ConditionCategory.FAIL,))
+
 @pytest.fixture(scope='session')
 def multi_index_dataframe():
     """Return a multi-indexed DataFrame."""
diff --git a/tests/suites/test_suites.py b/tests/suites/test_suites.py
--- a/tests/suites/test_suites.py
+++ b/tests/suites/test_suites.py
@@ -19,13 +19,10 @@
 import pandas as pd
 from sklearn.ensemble import AdaBoostClassifier
 from sklearn.model_selection import train_test_split
-from hamcrest.core.matcher import Matcher
-from hamcrest import assert_that, instance_of, only_contains, any_of
 from xgboost import XGBClassifier, XGBRegressor
 
-from deepchecks.core import SuiteResult, CheckResult, CheckFailure
-from deepchecks.tabular import suites, Dataset, SingleDatasetCheck, Suite
-from deepchecks.core.errors import DeepchecksBaseError
+from deepchecks.tabular import suites, Dataset
+from tests.conftest import get_expected_results_length, validate_suite_result
 
 
 @pytest.fixture()
@@ -133,44 +130,3 @@ def test_generic_custom(
         result = suite.run(**args)
         length = get_expected_results_length(suite, args)
         validate_suite_result(result, length)
-
-
-def validate_suite_result(
-    result: SuiteResult,
-    min_length: int,
-    exception_matcher: t.Optional[Matcher] = None
-):
-    assert_that(result, instance_of(SuiteResult))
-    assert_that(result.results, instance_of(list))
-    assert_that(len(result.results) >= min_length)
-
-    exception_matcher = exception_matcher or only_contains(instance_of(DeepchecksBaseError))
-
-    assert_that(result.results, only_contains(any_of( # type: ignore
-        instance_of(CheckFailure),
-        instance_of(CheckResult),
-    )))
-
-    failures = [
-        it.exception
-        for it in result.results
-        if isinstance(it, CheckFailure)
-    ]
-
-    if len(failures) != 0:
-        assert_that(actual=failures, matcher=exception_matcher) # type: ignore
-
-
-def get_expected_results_length(suite: Suite, args: t.Dict):
-    num_single = len([c for c in suite.checks.values() if isinstance(c, SingleDatasetCheck)])
-    num_others = len(suite.checks.values()) - num_single
-    multiply = 0
-    if 'train_dataset' in args:
-        multiply += 1
-    if 'test_dataset' in args:
-        multiply += 1
-    # If no train and no test (only model) there will be single result of check failure
-    if multiply == 0:
-        multiply = 1
-
-    return num_single * multiply + num_others
diff --git a/tests/vision/base/test_suite.py b/tests/vision/base/test_suite.py
--- a/tests/vision/base/test_suite.py
+++ b/tests/vision/base/test_suite.py
@@ -27,11 +27,13 @@
 
 from deepchecks.core import CheckResult, DatasetKind
 from deepchecks.core.errors import DeepchecksValueError, DatasetValidationError, DeepchecksNotSupportedError
+from deepchecks.vision.suites.default_suites import full_suite
 from deepchecks.vision.vision_data import TaskType
 from deepchecks.vision.base_checks import SingleDatasetCheck, TrainTestCheck, ModelOnlyCheck
 from deepchecks.vision.suite import Suite
 from deepchecks.vision.datasets.detection import coco
 from deepchecks.vision.datasets.classification import mnist
+from tests.conftest import get_expected_results_length, validate_suite_result
 
 
 def test_suite_execution():
@@ -61,9 +63,12 @@ def compute(self, context) -> CheckResult:
             return CheckResult(1)
 
     suite = Suite("test",
-                           DummyCheck(),
-                           DummyTrainTestCheck())
-    result = suite.run(train_dataset=coco_dataset, test_dataset=coco_dataset)
+                  DummyCheck(),
+                  DummyTrainTestCheck())
+    args = {'train_dataset': coco_dataset, 'test_dataset': coco_dataset}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
 
     assert_that(result.results[0].value, is_(0))
     assert_that(result.results[2].value, is_(1))
@@ -101,7 +106,11 @@ def compute(self, context) -> CheckResult:
     suite = Suite("test",
                   DummyCheck(),
                   DummyTrainTestCheck())
-    result = suite.run(train_dataset=coco_dataset, test_dataset=coco_dataset)
+
+    args = {'train_dataset': coco_dataset, 'test_dataset': coco_dataset}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
 
     assert_that(result.results[0].exception, instance_of(DeepchecksValueError))
     assert_that(result.results[0].exception.message, is_('bad init'))
@@ -142,7 +151,11 @@ def compute(self, context) -> CheckResult:
     suite = Suite("test",
                   DummyCheck(),
                   DummyTrainTestCheck())
-    result = suite.run(train_dataset=coco_dataset, test_dataset=coco_dataset)
+
+    args = {'train_dataset': coco_dataset, 'test_dataset': coco_dataset}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
 
     assert_that(result.results[0].exception, instance_of(DeepchecksValueError))
     assert_that(result.results[0].exception.message, is_('bad compute'))
@@ -194,10 +207,14 @@ def compute(self, context) -> CheckResult:
 
     suite = Suite("test",
                   DummyTrainTestCheck())
-    r = suite.run(train_dataset=coco_dataset)
 
-    assert_that(r.results[0].exception, instance_of(DeepchecksNotSupportedError))
-    assert_that(r.results[0].exception.message,
+    args = {'train_dataset': coco_dataset}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
+
+    assert_that(result.results[0].exception, instance_of(DeepchecksNotSupportedError))
+    assert_that(result.results[0].exception.message,
                 is_('Check is irrelevant if not supplied with both train and test datasets'))
 
     assert_that(executions, is_({'initialize_run': 1}))
@@ -216,6 +233,27 @@ def compute(self, context) -> CheckResult:
 
     suite = Suite("test",
                   DummyModelOnlyCheck())
-    suite.run(model="some model")
+
+    args = {'model': "some model"}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
 
     assert_that(executions, is_({'initialize_run': 1, 'compute': 1}))
+
+def test_full_suite_execution_mnist(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
+    suite = full_suite()
+    args = {'train_dataset': mnist_dataset_train, 'test_dataset': mnist_dataset_test,
+            'model':mock_trained_mnist, 'device': device}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)
+
+def test_full_suite_execution_coco(coco_train_visiondata, coco_test_visiondata,
+                                   mock_trained_yolov5_object_detection, device):
+    suite = full_suite()
+    args = {'train_dataset': coco_train_visiondata, 'test_dataset': coco_test_visiondata,
+            'model':mock_trained_yolov5_object_detection, 'device': device}
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)