Repository: deepchecks/deepchecks

Issue:
[FEAT] Attach exception traces to suite result. 
**Is your feature request related to a problem? Please describe.**
Currently it's often hard to debug exceptions in checks running as parts of suites - all you can see is the error message. 

**Describe the solution you'd like**
Attach the exception trace as part of the result object. 




Discussion:
as part of the result object, or as part of the display? The exception already exists as part of the object (CheckFailure has the exception inside)
If we want this to be displayed, then I suggest a hidden (collapsible) section where the traces are available to be looked at (e.g. an arrow at the end of each check of `checks that did not run` section so a user can look at the trace. @JKL98ISR  is this something that is possible?


close to impossible rn (the table is just a df). I think we can just add an error print function that loops over the checkfalures and shows the stack traces with the check title
suit_result.get_errors() and adding a print that tells the user that this exists?

Resolving patch:
diff --git a/deepchecks/core/check_result.py b/deepchecks/core/check_result.py
--- a/deepchecks/core/check_result.py
+++ b/deepchecks/core/check_result.py
@@ -501,9 +501,7 @@ def _get_metadata(self, with_doc_link: bool = False):
 
     def __repr__(self):
         """Return string representation."""
-        tb_str = traceback.format_exception(etype=type(self.exception), value=self.exception,
-                                            tb=self.exception.__traceback__)
-        return ''.join(tb_str)
+        return self.header + ': ' + str(self.exception)
 
     def _ipython_display_(self):
         """Display the check failure."""
@@ -513,3 +511,9 @@ def _ipython_display_(self):
             check_html += f'<p>{summary}</p>'
         check_html += f'<p style="color:red"> {self.exception}</p>'
         display_html(check_html, raw=True)
+
+    def print_traceback(self):
+        """Print the traceback of the failure."""
+        tb_str = traceback.format_exception(etype=type(self.exception), value=self.exception,
+                                            tb=self.exception.__traceback__)
+        print(''.join(tb_str))
diff --git a/deepchecks/core/display_suite.py b/deepchecks/core/display_suite.py
--- a/deepchecks/core/display_suite.py
+++ b/deepchecks/core/display_suite.py
@@ -43,6 +43,7 @@
 _CHECKS_WITH_CONDITIONS_TITLE = '<h2>Check With Conditions Output</h2>'
 _CHECKS_WITHOUT_CONDITIONS_TITLE = '<h2>Check Without Conditions Output</h2>'
 _CHECKS_WITHOUT_DISPLAY_TITLE = '<h2>Other Checks That Weren\'t Displayed</h2>'
+_FAILED_CHECKS_MESSAGE = '</br>To debug failed checks use get_failures() to get a list of CheckFailures</br>'
 
 
 def _get_check_widget(check_res: CheckResult, unique_id: str) -> widgets.VBox:
@@ -116,6 +117,7 @@ def _display_suite_widgets(summary: str,
                            checks_wo_conditions_display: List[CheckResult],
                            checks_w_condition_display: List[CheckResult],
                            others_table: List,
+                           contains_check_failures: bool,
                            light_hr: str,
                            html_out,
                            requirejs: bool = True):  # pragma: no cover
@@ -168,7 +170,8 @@ def _display_suite_widgets(summary: str,
         with warnings.catch_warnings():
             warnings.simplefilter(action='ignore', category=FutureWarning)
             others_df = dataframe_to_html(others_table.style.hide_index())
-        h2_widget = widgets.HTML(_CHECKS_WITHOUT_DISPLAY_TITLE)
+        debug_message = _FAILED_CHECKS_MESSAGE if contains_check_failures else ''
+        h2_widget = widgets.HTML(_CHECKS_WITHOUT_DISPLAY_TITLE + debug_message)
         others_tab.children = [h2_widget, _create_table_widget(others_df)]
     else:
         others_tab.children = [widgets.HTML(_NO_OUTPUT_TEXT)]
@@ -199,6 +202,7 @@ def _display_suite_no_widgets(summary: str,
                               checks_wo_conditions_display: List[CheckResult],
                               checks_w_condition_display: List[CheckResult],
                               others_table: List,
+                              contains_check_failures: bool,
                               light_hr: str):  # pragma: no cover
     """Display results of suite in IPython without widgets."""
     bold_hr = '<hr style="background-color: black;border: 0 none;color: black;height: 1px;">'
@@ -236,10 +240,12 @@ def _display_suite_no_widgets(summary: str,
         others_table.sort_values(by=['sort'], inplace=True)
         others_table.drop('sort', axis=1, inplace=True)
         others_h2 = f'{bold_hr}{_CHECKS_WITHOUT_DISPLAY_TITLE}'
+        debug_message = _FAILED_CHECKS_MESSAGE if contains_check_failures else ''
+
         with warnings.catch_warnings():
             warnings.simplefilter(action='ignore', category=FutureWarning)
             others_df = dataframe_to_html(others_table.style.hide_index())
-        display_html(others_h2 + others_df, raw=True)
+        display_html(others_h2 + debug_message + others_df, raw=True)
 
     if unique_id:
         display_html(f'<br><a href="#summary_{unique_id}" style="font-size: 14px">Go to top</a>', raw=True)
@@ -260,6 +266,7 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
     checks_wo_conditions_display: List[CheckResult] = []
     checks_w_condition_display: List[CheckResult] = []
     others_table = []
+    contains_failures = False
 
     for result in results:
         if isinstance(result, CheckResult):
@@ -272,6 +279,7 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
             if not result.have_display():
                 others_table.append([result.get_header(), 'Nothing found', 2])
         elif isinstance(result, CheckFailure):
+            contains_failures = True
             error_types = (
                 errors.DatasetValidationError,
                 errors.ModelValidationError,
@@ -337,6 +345,7 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
                                checks_wo_conditions_display,
                                checks_w_condition_display,
                                others_table,
+                               contains_failures,
                                light_hr,
                                html_out,
                                requirejs)
@@ -347,4 +356,5 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
                                   checks_wo_conditions_display,
                                   checks_w_condition_display,
                                   others_table,
+                                  contains_failures,
                                   light_hr)
diff --git a/deepchecks/core/suite.py b/deepchecks/core/suite.py
--- a/deepchecks/core/suite.py
+++ b/deepchecks/core/suite.py
@@ -13,7 +13,7 @@
 import abc
 import warnings
 from collections import OrderedDict
-from typing import Any, Union, List, Tuple
+from typing import Any, Union, List, Tuple, Dict
 
 from IPython.core.display import display_html
 from IPython.core.getipython import get_ipython
@@ -140,6 +140,20 @@ def to_wandb(self, dedicated_run: bool = None, **kwargs: Any):
         if dedicated_run:
             wandb.finish()
 
+    def get_failures(self) -> Dict[str, CheckFailure]:
+        """Get all the failed checks.
+
+        Returns
+        -------
+        Dict[str, CheckFailure]
+            All the check failures in the suite.
+        """
+        failures = {}
+        for res in self.results:
+            if isinstance(res, CheckFailure):
+                failures[res.header] = res
+        return failures
+
 
 class BaseSuite:
     """Class for running a set of checks together, and returning a unified pass / no-pass.


Test patch:
diff --git a/tests/base/check_suite_test.py b/tests/base/check_suite_test.py
--- a/tests/base/check_suite_test.py
+++ b/tests/base/check_suite_test.py
@@ -10,9 +10,9 @@
 #
 """suites tests"""
 import random
-from hamcrest import assert_that, calling, raises, equal_to, is_
+from hamcrest import assert_that, calling, raises, equal_to, is_, has_length, instance_of
 
-from deepchecks.core import CheckResult
+from deepchecks.core import CheckResult, CheckFailure
 from deepchecks.core.errors import DeepchecksValueError
 from deepchecks.tabular import Suite, SingleDatasetCheck, TrainTestCheck
 from deepchecks.tabular import checks as tabular_checks
@@ -147,3 +147,17 @@ def test_check_suite_instantiation_by_extending_another_check_suite():
         tabular_checks.MixedDataTypes,
         tabular_checks.PerformanceReport
     ]
+
+
+def test_get_error(iris_split_dataset_and_model_custom,
+                   diabetes_split_dataset_and_model_custom):
+    iris_train, iris_test, iris_model = iris_split_dataset_and_model_custom
+
+    suite = Suite(
+        "test",
+        tabular_checks.IsSingleValue(),
+        tabular_checks.ModelErrorAnalysis())
+
+    result = suite.run(train_dataset=iris_train, test_dataset=iris_test, model=iris_model)
+    assert_that(result.get_failures(), has_length(1))
+    assert_that(result.get_failures()['Model Error Analysis'], instance_of(CheckFailure))