Repository: cleanlab/cleanlab

Issue:
lab.find_issues(features=features) outputs error for underperforming issue
lab.find_issues(features=features) output
```
[/Users/sanjana/cleanlab_home/fork_cleanlab/cleanlab/datalab/internal/issue_finder.py:457](https://file+.vscode-resource.vscode-cdn.net/Users/sanjana/cleanlab_home/fork_cleanlab/cleanlab/datalab/internal/issue_finder.py:457): UserWarning: No labels were provided. The 'label' issue type will not be run.
  warnings.warn("No labels were provided. " "The 'label' issue type will not be run.")
Finding null issues ...
Finding outlier issues ...
Fitting OOD estimator based on provided features ...
Finding near_duplicate issues ...
Finding non_iid issues ...
Finding underperforming_group issues ...
Error in underperforming_group: UnderperformingGroupIssueManager.find_issues() missing 1 required positional argument: 'pred_probs'
Failed to check for these issue types: [UnderperformingGroupIssueManager]

Audit complete. 984 issues found in the dataset.
```
Dataset: https://www.kaggle.com/datasets/laotse/credit-risk-dataset/data

Code
```python
import pandas as pd
from cleanlab import Datalab
from sklearn.preprocessing import StandardScaler
import numpy as np

df = pd.read_csv("./credit_risk_dataset.csv")
df = df[~df.isnull().any(axis=1)].copy()
feature_columns = df.columns.to_list()
feature_columns.remove("loan_status")

X_raw = df[feature_columns]
labels = df["loan_status"]

cat_features = [
    "person_home_ownership",
    "loan_intent",
    "loan_grade",
    "cb_person_default_on_file",
]
numeric_features = [
    "person_age",
    "person_income",
    "person_emp_length",
    "loan_amnt",
    "loan_int_rate",
    "loan_percent_income",
    "cb_person_cred_hist_length",
]

X_encoded = pd.get_dummies(X_raw, columns=cat_features, drop_first=True, dtype='float')

scaler = StandardScaler()
X_processed = X_encoded.copy()
X_processed[numeric_features] = scaler.fit_transform(X_encoded[numeric_features])

lab = Datalab({"X": X_processed.to_numpy(), "y": labels})

lab.find_issues(features=X_processed.to_numpy())
```


Discussion:
@elisno seems like the mapping that decides what issue-types to run based on the supplied args is off. The Underperforming group check should only run if pred_probs were included in the supplied args.



Resolving patch:
diff --git a/cleanlab/datalab/internal/issue_finder.py b/cleanlab/datalab/internal/issue_finder.py
--- a/cleanlab/datalab/internal/issue_finder.py
+++ b/cleanlab/datalab/internal/issue_finder.py
@@ -41,15 +41,16 @@
 )
 from cleanlab.datalab.internal.model_outputs import (
     MultiClassPredProbs,
-    RegressionPredictions,
     MultiLabelPredProbs,
+    RegressionPredictions,
 )
 from cleanlab.datalab.internal.task import Task
 
 if TYPE_CHECKING:  # pragma: no cover
-    import numpy.typing as npt
     from typing import Callable
 
+    import numpy.typing as npt
+
     from cleanlab.datalab.datalab import Datalab
 
 
@@ -58,6 +59,7 @@
     "outlier": ["pred_probs", "features", "knn_graph"],
     "near_duplicate": ["features", "knn_graph"],
     "non_iid": ["pred_probs", "features", "knn_graph"],
+    # The underperforming_group issue type requires a pair of inputs: (pred_probs, <any_of_the_other_three>)
     "underperforming_group": ["pred_probs", "features", "knn_graph", "cluster_ids"],
     "data_valuation": ["features", "knn_graph"],
     "class_imbalance": [],
@@ -482,4 +484,16 @@ def get_available_issue_types(self, **kwargs):
         if drop_class_imbalance_check:
             issue_types_copy.pop("class_imbalance")
 
+        required_pairs_for_underperforming_group = [
+            ("pred_probs", "features"),
+            ("pred_probs", "knn_graph"),
+            ("pred_probs", "cluster_ids"),
+        ]
+        drop_underperforming_group_check = "underperforming_group" in issue_types_copy and not any(
+            all(key in kwargs and kwargs.get(key) is not None for key in pair)
+            for pair in required_pairs_for_underperforming_group
+        )
+        if drop_underperforming_group_check:
+            issue_types_copy.pop("underperforming_group")
+
         return issue_types_copy


Test patch:
diff --git a/tests/datalab/test_issue_finder.py b/tests/datalab/test_issue_finder.py
--- a/tests/datalab/test_issue_finder.py
+++ b/tests/datalab/test_issue_finder.py
@@ -1,9 +1,8 @@
-import pytest
 import numpy as np
-
-from cleanlab.datalab.internal.issue_finder import IssueFinder
+import pytest
 
 from cleanlab import Datalab
+from cleanlab.datalab.internal.issue_finder import IssueFinder
 from cleanlab.datalab.internal.task import Task
 
 
@@ -25,6 +24,101 @@ def issue_finder(self, lab):
     def test_init(self, issue_finder):
         assert issue_finder.verbosity == 1
 
+    @pytest.mark.parametrize("key", ["pred_probs", "features", "knn_graph"])
+    def test_get_available_issue_types_no_kwargs(self, issue_finder, key):
+        expected_issue_types = {"class_imbalance": {}}
+        issue_types = issue_finder.get_available_issue_types(**{key: None})
+        assert (
+            issue_types == expected_issue_types
+        ), "Only class_imbalance issue type for classification requires no kwargs"
+
+    @pytest.mark.parametrize(
+        "issue_types",
+        [
+            {"label": {}},
+            {"label": {"some_arg": "some_value"}},
+            {"label": {"some_arg": "some_value"}, "outlier": {}},
+            {"label": {}, "outlier": {}, "some_issue_type": {"some_arg": "some_value"}},
+            {},
+        ],
+    )
+    def test_get_available_issue_types_with_issue_types(self, issue_finder, issue_types):
+        available_issue_types = issue_finder.get_available_issue_types(issue_types=issue_types)
+        assert (
+            available_issue_types == issue_types
+        ), f"Failed to get available issue types with issue_types={issue_types}"
+
+    @pytest.mark.parametrize(
+        "keys, should_contain_underperforming_group",
+        [
+            # Test cases where 'pred_probs' is not provided, should all give False
+            (["features"], False),
+            (["knn_graph"], False),
+            (["cluster_ids"], False),
+            (["features", "knn_graph"], False),
+            (["features", "cluster_ids"], False),
+            (["knn_graph", "cluster_ids"], False),
+            (["features", "knn_graph", "cluster_ids"], False),
+            # Test cases where 'pred_probs' is provided should all give True
+            (["pred_probs", "features"], True),
+            (["pred_probs", "knn_graph"], True),
+            (["pred_probs", "cluster_ids"], True),
+            (["pred_probs", "features", "knn_graph"], True),
+            (["pred_probs", "features", "cluster_ids"], True),
+            (["pred_probs", "knn_graph", "cluster_ids"], True),
+            (["pred_probs", "features", "knn_graph", "cluster_ids"], True),
+            # only if other required keys are provided
+            (["pred_probs"], False),
+        ],
+        ids=lambda v: (
+            f"keys={v} "
+            if isinstance(v, list)
+            else ("> available" if v is True else "> unavailable")
+        ),
+    )
+    # Some warnings about preferring cluster_ids over knn_graph, or knn_graph over features can be ignored
+    @pytest.mark.filterwarnings(r"ignore:.*will (likely )?prefer.*:UserWarning")
+    # No other warnings should be allowed
+    @pytest.mark.filterwarnings("error")
+    def test_underperforming_group_availability_issue_1065(
+        self, issue_finder, keys, should_contain_underperforming_group
+    ):
+        """
+        Tests the availability of the 'underperforming_group' issue type based on the presence of 'pred_probs' and other required keys in the supplied arguments.
+
+        This test addresses issue #1065, where the mapping that decides which issue types to run based on the supplied arguments is incorrect.
+        Specifically, the 'underperforming_group' check should only be executed if 'pred_probs' and another required key are included in the supplied arguments.
+        See: https://github.com/cleanlab/cleanlab/issues/1065.
+
+        Parameters
+        ----------
+        keys : list
+            A list of keys to be included in the kwargs.
+        should_contain_underperforming_group : bool
+            A flag indicating whether the 'underperforming_group' issue type should be present in the available issue types.
+
+        Scenarios
+        ---------
+        Various combinations of 'features', 'pred_probs', 'knn_graph', and 'cluster_ids' are tested.
+
+        Asserts
+        -------
+        Ensures 'underperforming_group' is in the available issue types if 'pred_probs' and another required key are provided.
+        Ensures 'underperforming_group' is not in the available issue types if the required conditions are not met.
+        """
+        mock_value = object()  # Mock value to simulate presence of the required keys
+        kwargs = {key: mock_value for key in keys}
+
+        available_issue_types = issue_finder.get_available_issue_types(**kwargs)
+        if should_contain_underperforming_group:
+            assert (
+                "underperforming_group" in available_issue_types
+            ), "underperforming_group should be available if 'pred_probs' and another required key are provided"
+        else:
+            assert (
+                "underperforming_group" not in available_issue_types
+            ), "underperforming_group should not be available if the required conditions are not met"
+
     def test_get_available_issue_types(self, issue_finder):
         expected_issue_types = {"class_imbalance": {}}
         # Test with no kwargs, no issue type expected to be returned
@@ -47,6 +141,21 @@ def test_get_available_issue_types(self, issue_finder):
             fail_msg = f"Failed to get available issue types with issue_types={issue_types}"
             assert available_issue_types == issue_types, fail_msg
 
+        ## Test availability of underperforming_group issue type
+        only_features_available = {"features": np.random.random((10, 2))}
+        available_issue_types = issue_finder.get_available_issue_types(**only_features_available)
+        fail_msg = "underperforming_group should not be available if 'pred_probs' is not provided"
+        assert "underperforming_group" not in available_issue_types, fail_msg
+        features_and_pred_probs_available = {
+            **only_features_available,
+            "pred_probs": np.random.random((10, 2)),
+        }
+        available_issue_types = issue_finder.get_available_issue_types(
+            **features_and_pred_probs_available
+        )
+        fail_msg = "underperforming_group should be available if 'pred_probs' is provided"
+        assert "underperforming_group" in available_issue_types, fail_msg
+
     def test_find_issues(self, issue_finder, lab):
         N = len(lab.data)
         K = lab.get_info("statistics")["num_classes"]