Repository: cleanlab/cleanlab

Issue:
Extend label issue detection in Datalab to work even without pred_probs input
Goal: extend the label issue check in Datalab to work even if user only provided: `features`, `labels` to `Datalab.find_issues()`.

There are multiple ways this can be achieved:

Option 1 (easiest): Use sklearn `KNNclassifier` (or `LogisticRegression`) applied to `X=features, y=labels` in order to produce out-of-sample `pred_probs` and then continue as usual.

Option 2: Use methods from other papers like these (requires benchmarking them first):
- [SelfClean: A Self-Supervised Data Cleaning Strategy](https://arxiv.org/abs/2305.17048)
- [Detecting Corrupted Labels Without Training a Model to Predict](https://arxiv.org/abs/2110.06283)




Discussion:

Resolving patch:
diff --git a/cleanlab/datalab/issue_finder.py b/cleanlab/datalab/issue_finder.py
--- a/cleanlab/datalab/issue_finder.py
+++ b/cleanlab/datalab/issue_finder.py
@@ -209,13 +209,16 @@ def _resolve_required_args(self, pred_probs, features, knn_graph):
         features :
             Name of column containing precomputed embeddings.
 
+        knn_graph :
+            Sparse matrix representing distances between examples in the dataset in a k nearest neighbor graph.
+
         Returns
         -------
         args_dict :
             Dictionary of required arguments for each issue type, if available.
         """
         args_dict = {
-            "label": {"pred_probs": pred_probs},
+            "label": {"pred_probs": pred_probs, "features": features},
             "outlier": {"pred_probs": pred_probs, "features": features, "knn_graph": knn_graph},
             "near_duplicate": {"features": features, "knn_graph": knn_graph},
             "non_iid": {"features": features, "knn_graph": knn_graph},
diff --git a/cleanlab/datalab/issue_manager/label.py b/cleanlab/datalab/issue_manager/label.py
--- a/cleanlab/datalab/issue_manager/label.py
+++ b/cleanlab/datalab/issue_manager/label.py
@@ -17,6 +17,9 @@
 
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional
 
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.preprocessing import OneHotEncoder
+
 import numpy as np
 
 from cleanlab.classification import CleanLearning
@@ -25,7 +28,7 @@
 
 if TYPE_CHECKING:  # pragma: no cover
     import pandas as pd
-
+    import numpy.typing as npt
     from cleanlab.datalab.datalab import Datalab
 
 
@@ -37,6 +40,10 @@ class LabelIssueManager(IssueManager):
     datalab :
         A Datalab instance.
 
+    k :
+        The number of nearest neighbors to consider when computing pred_probs from features.
+        Only applicable if features are provided and pred_probs are not.
+
     clean_learning_kwargs :
         Keyword arguments to pass to the :py:meth:`CleanLearning <cleanlab.classification.CleanLearning>` constructor.
 
@@ -61,12 +68,14 @@ class LabelIssueManager(IssueManager):
     def __init__(
         self,
         datalab: Datalab,
+        k: int = 10,
         clean_learning_kwargs: Optional[Dict[str, Any]] = None,
         health_summary_parameters: Optional[Dict[str, Any]] = None,
         **_,
     ):
         super().__init__(datalab)
         self.cl = CleanLearning(**(clean_learning_kwargs or {}))
+        self.k = k
         self.health_summary_parameters: Dict[str, Any] = (
             health_summary_parameters.copy() if health_summary_parameters else {}
         )
@@ -118,9 +127,39 @@ def _reset(self) -> None:
 
     def find_issues(
         self,
-        pred_probs: np.ndarray,
+        pred_probs: Optional[npt.NDArray] = None,
+        features: Optional[npt.NDArray] = None,
         **kwargs,
     ) -> None:
+        """Find label issues in the datalab.
+
+        Parameters
+        ----------
+        pred_probs :
+            The predicted probabilities for each example.
+
+        features :
+            The features for each example.
+        """
+        if pred_probs is None:
+            if features is None:
+                raise ValueError(
+                    "Either pred_probs or features must be provided to find label issues."
+                )
+            # produce out-of-sample pred_probs from features
+            knn = KNeighborsClassifier(n_neighbors=self.k + 1)
+            knn.fit(features, self.datalab.labels)
+            pred_probs = knn.predict_proba(features)
+
+            encoder = OneHotEncoder()
+            label_transform = self.datalab.labels.reshape(-1, 1)
+            one_hot_label = encoder.fit_transform(label_transform)
+
+            # adjust pred_probs so it is out-of-sample
+            pred_probs = np.asarray(
+                (pred_probs - 1 / (self.k + 1) * one_hot_label) * (self.k + 1) / self.k
+            )
+
         self.health_summary_parameters.update({"pred_probs": pred_probs})
         # Find examples with label issues
         self.issues = self.cl.find_label_issues(


Test patch:
diff --git a/tests/datalab/test_datalab.py b/tests/datalab/test_datalab.py
--- a/tests/datalab/test_datalab.py
+++ b/tests/datalab/test_datalab.py
@@ -809,8 +809,8 @@ def test_incremental_search(self, pred_probs, random_embeddings):
         lab = Datalab(data=data, label_name="labels")
         lab.find_issues(features=random_embeddings)
         summary = lab.get_issue_summary()
-        assert len(summary) == 3
-        assert "label" not in summary["issue_type"].values
+        assert len(summary) == 4
+        assert "label" in summary["issue_type"].values
         lab.find_issues(pred_probs=pred_probs, issue_types={"label": {}})
         summary = lab.get_issue_summary()
         assert len(summary) == 4
@@ -818,6 +818,38 @@ def test_incremental_search(self, pred_probs, random_embeddings):
         label_summary = lab.get_issue_summary("label")
         assert label_summary["num_issues"].values[0] > 0
 
+    def test_build_pred_probs_from_features(self, random_embeddings):
+        data = {"labels": np.random.randint(0, 2, 100)}
+        lab = Datalab(data=data, label_name="labels")
+        lab.find_issues(features=random_embeddings, issue_types={"label": {}})
+        summary = lab.get_issue_summary()
+        assert len(summary) == 1
+        assert "label" in summary["issue_type"].values
+        lab.find_issues(features=random_embeddings, issue_types={"label": {"k": 5}})
+        summary = lab.get_issue_summary()
+        assert len(summary) == 1
+        assert "label" in summary["issue_type"].values
+
+    def test_pred_probs_precedence(self, pred_probs, random_embeddings):
+        data = {"labels": np.random.randint(0, 2, 100)}
+        lab = Datalab(data=data, label_name="labels")
+        lab.find_issues(pred_probs=pred_probs, issue_types={"label": {}})
+        summary = lab.get_issue_summary()
+        assert "label" in summary["issue_type"].values
+        label_summary_pred_probs = lab.get_issue_summary("label")
+        assert label_summary_pred_probs["num_issues"].values[0] > 0
+        lab = Datalab(data=data, label_name="labels")
+        lab.find_issues(
+            features=random_embeddings, pred_probs=pred_probs, issue_types={"label": {}}
+        )
+        summary = lab.get_issue_summary()
+        assert "label" in summary["issue_type"].values
+        label_summary_both = lab.get_issue_summary("label")
+        assert (
+            label_summary_both["num_issues"].values[0]
+            == label_summary_pred_probs["num_issues"].values[0]
+        )
+
 
 class TestDatalabFindOutlierIssues:
     @pytest.fixture
@@ -930,5 +962,6 @@ def test_find_issues_features_works_with_and_without_labels(self, features, labe
         issues_with_labels = lab_with_labels.issues
         issues_without_label_name = lab_without_label_name.issues
 
-        pd.testing.assert_frame_equal(issues_without_labels, issues_with_labels)
+        # issues_with_labels should have two additional columns about label issues
+        assert len(issues_without_labels.columns) + 2 == len(issues_with_labels.columns)
         pd.testing.assert_frame_equal(issues_without_labels, issues_without_label_name)