Repository: cleanlab/cleanlab

Issue:
Add underperforming_group issue type among the Datalab defaults 
Test issue manager with different datasets (Image, tabular etc.) to make sure that the underperforming group in the dataset is extracted successfully.  List any failure cases that might need to be addressed before adding this issue type to the defaults.


Discussion:
first add extra step that `is_issue = True` only if the cluster in question contains at least M datapoints, for say M = 10

(maybe this is already done though, the responsibility for this should really lie with the clustering algorithm to never produce small clusters in the first place)
dont forget to update the datalab documentation listing the default issue types

Resolving patch:
diff --git a/cleanlab/datalab/internal/issue_manager_factory.py b/cleanlab/datalab/internal/issue_manager_factory.py
--- a/cleanlab/datalab/internal/issue_manager_factory.py
+++ b/cleanlab/datalab/internal/issue_manager_factory.py
@@ -223,6 +223,7 @@ def list_default_issue_types(task: str) -> List[str]:
             "near_duplicate",
             "non_iid",
             "class_imbalance",
+            "underperforming_group",
         ],
         "regression": [
             "null",


Test patch:
diff --git a/tests/datalab/datalab/test_datalab.py b/tests/datalab/datalab/test_datalab.py
--- a/tests/datalab/datalab/test_datalab.py
+++ b/tests/datalab/datalab/test_datalab.py
@@ -89,6 +89,7 @@ def test_list_default_issue_types(self):
             "near_duplicate",
             "non_iid",
             "class_imbalance",
+            "underperforming_group",
         ]
 
     def tmp_path(self):
diff --git a/tests/datalab/test_cleanvision_integration.py b/tests/datalab/test_cleanvision_integration.py
--- a/tests/datalab/test_cleanvision_integration.py
+++ b/tests/datalab/test_cleanvision_integration.py
@@ -32,7 +32,7 @@ def num_imagelab_issues(self):
 
     @pytest.fixture
     def num_datalab_issues(self):
-        return 5
+        return 6
 
     @pytest.fixture
     def pred_probs(self, image_dataset):
@@ -69,6 +69,7 @@ def test_imagelab_issues_checked(
             "near_duplicate",
             "class_imbalance",
             "null",
+            "underperforming_group",
             # "non_iid",
         ]
 
@@ -94,12 +95,14 @@ def test_imagelab_issues_checked(
                     "near_duplicate",
                     "class_imbalance",
                     "null",
+                    "underperforming_group",
                 ],
-                "num_issues": [1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0],
+                "num_issues": [1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
             }
         )
         expected_count = df.sort_values(by="issue_type")["num_issues"].tolist()
         count = datalab.issue_summary.sort_values(by="issue_type")["num_issues"].tolist()
+        assert set(datalab.issue_summary["issue_type"].tolist()) == set(df["issue_type"].tolist())
         assert count == expected_count
         assert datalab.issue_summary["num_issues"].sum() == df["num_issues"].sum()
 
@@ -147,7 +150,15 @@ def test_imagelab_issues_not_checked(
         assert len(datalab.issues.columns) == num_datalab_issues * 2
         assert len(datalab.issue_summary) == num_datalab_issues
 
-        all_keys = ["statistics", "label", "outlier", "near_duplicate", "class_imbalance", "null"]
+        all_keys = [
+            "statistics",
+            "label",
+            "outlier",
+            "near_duplicate",
+            "class_imbalance",
+            "null",
+            "underperforming_group",
+        ]
 
         assert set(all_keys) == set(datalab.info.keys())
         datalab.report()