Repository: deepchecks/deepchecks

Issue:
[BUG] Check display order in suites
**Describe the bug**
Check displays don't appear in the same order as in table, in train_test_validation suite.

**To Reproduce**
Run the train_test_validation on the lending club dataset
See outputs:
![image](https://user-images.githubusercontent.com/33841818/170472123-14641fc9-9a5e-45e2-88d9-b20047d89cea.png)

**Expected behavior**
Check display order should fit the order in the conditions summary (if there are any failures/warning they should be in that order)


Discussion:

Resolving patch:
diff --git a/deepchecks/core/check_result.py b/deepchecks/core/check_result.py
--- a/deepchecks/core/check_result.py
+++ b/deepchecks/core/check_result.py
@@ -172,8 +172,9 @@ def priority(self) -> int:
 
         * if at least one condition did not pass and is of category 'FAIL', return 1.
         * if at least one condition did not pass and is of category 'WARN', return 2.
-        * if check result do not have assigned conditions, return 3.
+        * if at least one condition did not pass and is of category 'ERROR', return 3.
         * if all conditions passed, return 4.
+        * if check result do not have assigned conditions, return 5.
 
         Returns
         -------
@@ -181,13 +182,15 @@ def priority(self) -> int:
             priority of the check result.
         """
         if not self.have_conditions:
-            return 3
+            return 5
 
         for c in self.conditions_results:
             if c.is_pass is False and c.category == ConditionCategory.FAIL:
                 return 1
             if c.is_pass is False and c.category == ConditionCategory.WARN:
                 return 2
+            if c.is_pass is False and c.category == ConditionCategory.ERROR:
+                return 3
 
         return 4
 
diff --git a/deepchecks/core/serialization/common.py b/deepchecks/core/serialization/common.py
--- a/deepchecks/core/serialization/common.py
+++ b/deepchecks/core/serialization/common.py
@@ -47,7 +47,7 @@
     'plotlyjs_script',
     'requirejs_script',
     'flatten',
-    'join'
+    'join',
 ]
 
 
@@ -131,6 +131,7 @@ def aggregate_conditions(
     pd.Dataframe:
         the condition table.
     """
+    # NOTE: if you modified this function also modify 'sort_check_results'
     check_results = (
         [check_results]
         if isinstance(check_results, check_types.CheckResult)
@@ -159,11 +160,11 @@ def aggregate_conditions(
 
     df = pd.DataFrame(
         data=data,
-        columns=['Status', 'Check', 'Condition', 'More Info', 'sort']
+        columns=['Status', 'Check', 'Condition', 'More Info', '$priority']
     )
 
-    df.sort_values(by=['sort'], inplace=True)
-    df.drop('sort', axis=1, inplace=True)
+    df.sort_values(by=['$priority'], inplace=True)
+    df.drop('$priority', axis=1, inplace=True)
 
     if include_check_name is False:
         df.drop('Check', axis=1, inplace=True)
diff --git a/deepchecks/core/suite.py b/deepchecks/core/suite.py
--- a/deepchecks/core/suite.py
+++ b/deepchecks/core/suite.py
@@ -14,7 +14,7 @@
 import io
 import warnings
 from collections import OrderedDict
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import jsonpickle
 from IPython.core.display import display, display_html
@@ -51,7 +51,7 @@ class SuiteResult:
     def __init__(self, name: str, results, extra_info: Optional[List[str]] = None):
         """Initialize suite result."""
         self.name = name
-        self.results = results
+        self.results = sort_check_results(results)
         self.extra_info = extra_info or []
 
         # TODO: add comment about code below
@@ -394,3 +394,33 @@ def remove(self, index: int):
             raise DeepchecksValueError(f'No index {index} in suite')
         self.checks.pop(index)
         return self
+
+
+def sort_check_results(
+    check_results: Sequence[BaseCheckResult]
+) -> List[BaseCheckResult]:
+    """Sort sequence of 'CheckResult' instances.
+
+    Returns
+    -------
+    List[check_types.CheckResult]
+    """
+    order = []
+    check_results_index = {}
+
+    for index, it in enumerate(check_results):
+        check_results_index[index] = it
+
+        if isinstance(it, CheckResult):
+            order.append((it.priority, index))
+        elif isinstance(it, CheckFailure):
+            order.append((998, index))
+        else:
+            order.append((999, index))
+
+    order = sorted(order)
+
+    return [
+        check_results_index[index]
+        for _, index in order
+    ]


Test patch:
diff --git a/tests/tabular/checks/train_test_validation/string_mismatch_comparison_test.py b/tests/tabular/checks/train_test_validation/string_mismatch_comparison_test.py
--- a/tests/tabular/checks/train_test_validation/string_mismatch_comparison_test.py
+++ b/tests/tabular/checks/train_test_validation/string_mismatch_comparison_test.py
@@ -11,7 +11,7 @@
 """Contains unit tests for the string_mismatch check."""
 import numpy as np
 import pandas as pd
-from hamcrest import assert_that, has_entries, has_entry, has_items, has_length, equal_to
+from hamcrest import assert_that, equal_to, has_entries, has_entry, has_items, has_length
 
 from deepchecks.tabular.checks import StringMismatchComparison
 from deepchecks.tabular.dataset import Dataset
diff --git a/tests/tabular/test_dummy_model.py b/tests/tabular/test_dummy_model.py
--- a/tests/tabular/test_dummy_model.py
+++ b/tests/tabular/test_dummy_model.py
@@ -12,23 +12,19 @@
 from hamcrest import assert_that, calling, has_items, raises
 from sklearn.linear_model import LogisticRegression
 from sklearn.model_selection import train_test_split
-from deepchecks.core.check_result import CheckResult
 
+from deepchecks.core.check_result import CheckResult
 from deepchecks.core.condition import ConditionCategory
 from deepchecks.core.errors import DeepchecksValueError, ValidationError
 from deepchecks.tabular.base_checks import TrainTestCheck
-from deepchecks.tabular.checks.model_evaluation.model_error_analysis import \
-    ModelErrorAnalysis
-from deepchecks.tabular.checks.model_evaluation.regression_error_distribution import \
-    RegressionErrorDistribution
+from deepchecks.tabular.checks.model_evaluation.model_error_analysis import ModelErrorAnalysis
+from deepchecks.tabular.checks.model_evaluation.regression_error_distribution import RegressionErrorDistribution
 from deepchecks.tabular.checks.model_evaluation.roc_report import RocReport
-from deepchecks.tabular.checks.model_evaluation.simple_model_comparison import \
-    SimpleModelComparison
+from deepchecks.tabular.checks.model_evaluation.simple_model_comparison import SimpleModelComparison
 from deepchecks.tabular.context import Context
 from deepchecks.tabular.dataset import Dataset
 from tests.base.utils import equal_condition_result
-from tests.tabular.checks.model_evaluation.simple_model_comparison_test import \
-    assert_regression
+from tests.tabular.checks.model_evaluation.simple_model_comparison_test import assert_regression
 
 
 def _dummify_model(train, test, model):
@@ -83,7 +79,7 @@ def test_roc_condition_ratio_more_than_passed(iris_clean):
     y_pred_train, y_pred_test, y_proba_train, y_proba_test = _dummify_model(ds, None, clf)
 
     check = RocReport().add_condition_auc_not_less_than()
-    result = check.conditions_decision(check.run(ds,           
+    result = check.conditions_decision(check.run(ds,
                                                  y_pred_train=y_pred_train, y_pred_test=y_pred_test,
                                                  y_proba_train=y_proba_train, y_proba_test=y_proba_test))
 
@@ -105,7 +101,7 @@ def test_regression_error_absolute_kurtosis_not_greater_than_not_passed(diabetes
     check = RegressionErrorDistribution().add_condition_kurtosis_not_less_than()
 
     # Act
-    result = check.conditions_decision(check.run(test,           
+    result = check.conditions_decision(check.run(test,
                                                  y_pred_train=y_pred_train, y_pred_test=y_pred_test,
                                                  y_proba_train=y_proba_train, y_proba_test=y_proba_test))
 
@@ -124,7 +120,7 @@ def test_simple_model_comparison_regression_random_state(diabetes_split_dataset_
     y_pred_train, y_pred_test, y_proba_train, y_proba_test = _dummify_model(train_ds, test_ds, clf)
     check = SimpleModelComparison(simple_model_type='random', random_state=0)
     # Act X
-    result = check.run(train_ds, test_ds, 
+    result = check.run(train_ds, test_ds,
                        y_pred_train=y_pred_train, y_pred_test=y_pred_test,
                        y_proba_train=y_proba_train, y_proba_test=y_proba_test).value
     # Assert
@@ -145,7 +141,7 @@ def run_logic(self, context: Context) -> CheckResult:
     # Act
     assert_that(
         calling(NewDataCheck().run)
-            .with_args(train_dataset=train, test_dataset=test,          
+            .with_args(train_dataset=train, test_dataset=test,
                        y_pred_train=y_pred_train, y_pred_test=y_pred_test,
                        y_proba_train=y_proba_train, y_proba_test=y_proba_test),
         raises(
diff --git a/tests/utils/correlation_methods_test.py b/tests/utils/correlation_methods_test.py
--- a/tests/utils/correlation_methods_test.py
+++ b/tests/utils/correlation_methods_test.py
@@ -10,7 +10,7 @@
 #
 
 import pandas as pd
-from hamcrest import (assert_that, close_to)
+from hamcrest import assert_that, close_to
 
 from deepchecks.utils import correlation_methods