Repository: cleanlab/cleanlab

Issue:
add `show_all_issues` optional argument to: datalab.report()
This arg has default = False. 
If True, the report includes issue types with `num_issue = 0` in the Datalab report, otherwise we suppress such issues from the report.
This is critical to ensure the report doesn't get overwhelming as more issue-types are supported 


Also: `datalab.report()` docstring is missing documentation for the argument:
`show_summary_score` 

Add it while adding this other argument.
add `show_all_issues` optional argument to: datalab.report()
This arg has default = False. 
If True, the report includes issue types with `num_issue = 0` in the Datalab report, otherwise we suppress such issues from the report.
This is critical to ensure the report doesn't get overwhelming as more issue-types are supported 


Also: `datalab.report()` docstring is missing documentation for the argument:
`show_summary_score` 

Add it while adding this other argument.


Discussion:
Hi @jwmueller I am working on this ✌️
Hi @jwmueller I am working on this ✌️

Resolving patch:
diff --git a/cleanlab/datalab/datalab.py b/cleanlab/datalab/datalab.py
--- a/cleanlab/datalab/datalab.py
+++ b/cleanlab/datalab/datalab.py
@@ -377,6 +377,7 @@ def report(
         verbosity: Optional[int] = None,
         include_description: bool = True,
         show_summary_score: bool = False,
+        show_all_issues: bool = False,
     ) -> None:
         """Prints informative summary of all issues.
 
@@ -393,6 +394,13 @@ def report(
             Whether or not to include a description of each issue type in the report.
             Consider setting this to ``False`` once you're familiar with how each issue type is defined.
 
+        show_summary_score :
+            Whether or not to include the overall severity of each issue type in the report.
+
+        show_all_issues :
+            Whether or not to show all issues in the report, or only the issues for which examples were found in the dataset
+            With this set to ``True``, the report may include more types of issues that were not detected in the dataset.
+
         See Also
         --------
         For advanced usage, see documentation for the
@@ -410,6 +418,7 @@ def report(
             verbosity=verbosity,
             include_description=include_description,
             show_summary_score=show_summary_score,
+            show_all_issues=show_all_issues,
             imagelab=self._imagelab,
         )
         reporter.report(num_examples=num_examples)
diff --git a/cleanlab/datalab/internal/adapter/imagelab.py b/cleanlab/datalab/internal/adapter/imagelab.py
--- a/cleanlab/datalab/internal/adapter/imagelab.py
+++ b/cleanlab/datalab/internal/adapter/imagelab.py
@@ -149,6 +149,7 @@ def __init__(
         verbosity: int = 1,
         include_description: bool = True,
         show_summary_score: bool = False,
+        show_all_issues: bool = False,
     ):
         super().__init__(
             data_issues=data_issues,
@@ -156,6 +157,7 @@ def __init__(
             verbosity=verbosity,
             include_description=include_description,
             show_summary_score=show_summary_score,
+            show_all_issues=show_all_issues,
         )
         self.imagelab = imagelab
 
diff --git a/cleanlab/datalab/internal/report.py b/cleanlab/datalab/internal/report.py
--- a/cleanlab/datalab/internal/report.py
+++ b/cleanlab/datalab/internal/report.py
@@ -63,6 +63,7 @@ def __init__(
         verbosity: int = 1,
         include_description: bool = True,
         show_summary_score: bool = False,
+        show_all_issues: bool = False,
         **kwargs,
     ):
         self.data_issues = data_issues
@@ -70,6 +71,20 @@ def __init__(
         self.verbosity = verbosity
         self.include_description = include_description
         self.show_summary_score = show_summary_score
+        self.show_all_issues = show_all_issues
+
+    def _get_empty_report(self) -> str:
+        """This method is used to return a report when there are
+        no issues found in the data with Datalab.find_issues().
+        """
+        report_str = "No issues found in the data. Good job!"
+        if not self.show_summary_score:
+            recommendation_msg = (
+                "Try re-running Datalab.report() with "
+                "`show_summary_score = True` and `show_all_issues = True`."
+            )
+            report_str += f"\n\n{recommendation_msg}"
+        return report_str
 
     def report(self, num_examples: int) -> None:
         """Prints a report about identified issues in the data.
@@ -104,11 +119,28 @@ def get_report(self, num_examples: int) -> str:
         """
         report_str = ""
         issue_summary = self.data_issues.issue_summary
+        should_return_empty_report = not (
+            self.show_all_issues or issue_summary.empty or issue_summary["num_issues"].sum() > 0
+        )
+
+        if should_return_empty_report:
+            return self._get_empty_report()
         issue_summary_sorted = issue_summary.sort_values(by="num_issues", ascending=False)
         report_str += self._write_summary(summary=issue_summary_sorted)
 
         issue_types = self._get_issue_types(issue_summary_sorted)
 
+        def add_issue_to_report(issue_name: str) -> bool:
+            """Returns True if the issue should be added to the report.
+            It is excluded if show_all_issues is False and there are no issues of that type
+            found in the data.
+            """
+            if self.show_all_issues:
+                return True
+            summary = self.data_issues.get_issue_summary(issue_name=issue_name)
+            has_issues = summary["num_issues"][0] > 0
+            return has_issues
+
         issue_reports = [
             _IssueManagerFactory.from_str(issue_type=key, task=self.task).report(
                 issues=self.data_issues.get_issues(issue_name=key),
@@ -154,7 +186,10 @@ def _write_summary(self, summary: pd.DataFrame) -> str:
     def _get_issue_types(self, issue_summary: pd.DataFrame) -> List[str]:
         issue_types = [
             issue_type
-            for issue_type in issue_summary["issue_type"].tolist()
+            for issue_type, num_issues in zip(
+                issue_summary["issue_type"].tolist(), issue_summary["num_issues"].tolist()
+            )
             if issue_type not in DEFAULT_CLEANVISION_ISSUES
+            and (self.show_all_issues or num_issues > 0)
         ]
         return issue_types


Test patch:
diff --git a/tests/datalab/test_datalab.py b/tests/datalab/test_datalab.py
--- a/tests/datalab/test_datalab.py
+++ b/tests/datalab/test_datalab.py
@@ -1786,3 +1786,51 @@ def test_underperforming_group_reuses_knn_graph(self, features, pred_probs, labe
         assert (
             time_underperforming_after_outlier < time_only_underperforming_group
         ), "KNN graph reuse should make this run of find_issues faster."
+
+
+class TestDatalabDefaultReporting:
+    """This test class focuses on testing the default behavior of the reporting functionality.
+
+    If there are no issues found, the report should contain a message for no issues found.
+
+    If there are issues found, the report should start with a summary of the issues found.
+
+    Other test classes focus on testing the reporting functionality with different issue types.
+    """
+
+    @pytest.fixture
+    def data(self):
+        np.random.seed(SEED)
+        X = np.random.rand(100, 10)
+        y = np.random.randint(0, 2, 100)
+
+        X[y == 1] += 1.5
+        return {"X": X, "y": y}
+
+    def test_report(self, data):
+        lab = Datalab(data=data, label_name="y")
+        lab.find_issues(features=data["X"], issue_types={"label": {}})
+        with contextlib.redirect_stdout(io.StringIO()) as f:
+            lab.report()
+        report = f.getvalue()
+        assert (
+            "No issues found in the data." in report
+        ), "Report should contain a message for no issues found"
+
+    def test_report_with_one_label_issue(self, data):
+        # Flip the label of one example
+        y = data["y"]
+        y[-1] = 1 - y[-1]
+
+        lab = Datalab(data={"X": data["X"], "y": y}, label_name="y")
+        lab.find_issues(features=data["X"], issue_types={"label": {}})
+        with contextlib.redirect_stdout(io.StringIO()) as f:
+            lab.report()
+        report = f.getvalue()
+        expected_header = (
+            "Here is a summary of the different kinds of issues found in the data:"
+            "\n\nissue_type  num_issues\n     label           1\n\n"
+        )
+        assert report.startswith(
+            expected_header
+        ), "Report should contain a message for one issue found"
diff --git a/tests/datalab/test_report.py b/tests/datalab/test_report.py
--- a/tests/datalab/test_report.py
+++ b/tests/datalab/test_report.py
@@ -100,3 +100,123 @@ def from_str(*args, **kwargs):
         report = reporter.get_report(num_examples=3)
         expected_report = "\n\n".join(["Here is a lab summary", "foo report"])
         assert report == expected_report
+
+    @pytest.mark.parametrize(
+        "show_all_issues, expected_report",
+        [
+            (True, "Here is a lab summary\n\nfoo report\n\n\nbar report"),
+            (False, "Here is a lab summary\n\nfoo report"),
+        ],
+    )
+    def test_show_all_issues(
+        self, reporter, data_issues, monkeypatch, show_all_issues, expected_report
+    ):
+        """Test that the report method works. Assuming we have two issue managers, each should add
+        their section to the report."""
+
+        mock_issue_manager_foo = Mock()
+        mock_issue_manager_foo.issue_name = "foo"
+        mock_issue_manager_foo.report.return_value = "foo report"
+
+        mock_issue_manager_bar = Mock()
+        mock_issue_manager_bar.issue_name = "bar"
+        mock_issue_manager_bar.report.return_value = "bar report"
+
+        class MockIssueManagerFactory:
+            @staticmethod
+            def from_str(*args, **kwargs):
+                name = kwargs["issue_type"]
+                issue_managers = {
+                    "foo": mock_issue_manager_foo,
+                    "bar": mock_issue_manager_bar,
+                }
+                issue_manager = issue_managers.get(name)
+                if issue_manager is None:
+                    raise ValueError(f"Unknown issue manager name: {name}")
+                return issue_manager
+
+        monkeypatch.setattr(
+            "cleanlab.datalab.internal.report._IssueManagerFactory", MockIssueManagerFactory
+        )
+        mock_issues = pd.DataFrame(
+            {
+                "is_foo_issue": [False, True, False, False, False],
+                "foo_score": [0.6, 0.2, 0.7, 0.7, 0.8],
+                "is_bar_issue": [False, False, False, False, False],
+                "bar_score": [0.7, 0.9, 0.8, 0.8, 0.8],
+            }
+        )
+        monkeypatch.setattr(data_issues, "issues", mock_issues)
+
+        # "bar" issue may be omitted in report, unless show_all_issues is True
+        mock_issue_summary = pd.DataFrame(
+            {
+                "issue_type": ["foo", "bar"],
+                "score": [0.6, 0.8],
+                "num_issues": [1, 0],
+            }
+        )
+
+        mock_info = {
+            "foo": {"foobar": "baz"},
+            "bar": {"barfoo": "bazbar"},
+        }
+
+        monkeypatch.setattr(data_issues, "issue_summary", mock_issue_summary)
+
+        reporter = Reporter(
+            data_issues=data_issues,
+            task="classification",
+            verbosity=0,
+            include_description=False,
+            show_all_issues=show_all_issues,
+        )
+        monkeypatch.setattr(data_issues, "issues", mock_issues, raising=False)
+        monkeypatch.setattr(data_issues, "info", mock_info, raising=False)
+
+        monkeypatch.setattr(
+            reporter, "_write_summary", lambda *args, **kwargs: "Here is a lab summary\n\n"
+        )
+        report = reporter.get_report(num_examples=3)
+        assert report == expected_report
+
+    summary = pd.DataFrame(
+        {
+            "issue_type": ["foo", "bar"],
+            "score": [0.6, 0.8],
+            "num_issues": [1, 0],
+        }
+    )
+
+    def test_summary_with_score(self, reporter, data_issues, monkeypatch):
+        """Test that the _write_summary method returns the expected output when show_summary_score is True.
+
+        It should include the score column in the summary and a note about what the score means.
+        """
+        mock_statistics = {"num_examples": 100, "num_classes": 5}
+        monkeypatch.setattr(data_issues, "get_info", lambda *args, **kwargs: mock_statistics)
+
+        expected_output = (
+            "Here is a summary of the different kinds of issues found in the data:\n\n"
+            + self.summary.to_string(index=False)
+            + "\n\n"
+            + "(Note: A lower score indicates a more severe issue across all examples in the dataset.)\n\n"
+            + "Dataset Information: num_examples: 100, num_classes: 5\n\n\n"
+        )
+
+        reporter.show_summary_score = True
+        assert reporter._write_summary(self.summary) == expected_output
+
+    def test_summary_without_score(self, reporter, data_issues, monkeypatch):
+        mock_statistics = {"num_examples": 100, "num_classes": 5}
+        monkeypatch.setattr(data_issues, "get_info", lambda *args, **kwargs: mock_statistics)
+
+        expected_output = (
+            "Here is a summary of the different kinds of issues found in the data:\n\n"
+            + self.summary.drop(columns=["score"]).to_string(index=False)
+            + "\n\n"
+            + "Dataset Information: num_examples: 100, num_classes: 5\n\n\n"
+        )
+
+        reporter.show_summary_score = False
+        assert reporter._write_summary(self.summary) == expected_output