Repository: cleanlab/cleanlab

Issue:
test_scores_for_identical_examples unit test fails
see logs from here:  https://github.com/cleanlab/cleanlab/actions/runs/8455871879/job/23167543959

![Screen Shot 2024-03-29 at 4 02 43 PM](https://github.com/cleanlab/cleanlab/assets/1390638/97e90bc7-6817-410c-b549-aea1d30572e7)



Discussion:

Resolving patch:
diff --git a/cleanlab/internal/outlier.py b/cleanlab/internal/outlier.py
--- a/cleanlab/internal/outlier.py
+++ b/cleanlab/internal/outlier.py
@@ -18,6 +18,8 @@
 Helper functions used internally for outlier detection tasks.
 """
 
+from typing import Union
+
 import numpy as np
 
 
@@ -67,9 +69,56 @@ def transform_distances_to_scores(
     array([0.88988177, 0.80519832])
     """
     # Map ood_features_scores to range 0-1 with 0 = most concerning
-    ood_features_scores: np.ndarray = np.exp(-1 * avg_distances / scaling_factor * t)
+    return np.exp(-1 * avg_distances / scaling_factor * t)
+
+
+def correct_precision_errors(
+    scores: np.ndarray,
+    avg_distances: np.ndarray,
+    metric: str,
+    C: int = 100,
+    p: Union[int, None] = None,
+):
+    """
+    Ensure that scores where avg_distances are below the tolerance threshold get a score of one.
+
+    Parameters
+    ----------
+    scores : np.ndarray
+        An array of scores of shape ``(N)``, where N is the number of examples.
+        Each entry represents a score between 0 and 1.
+
+    avg_distances : np.ndarray
+        An array of distances of shape ``(N)``, where N is the number of examples.
+        Each entry represents an example's average distance to its k nearest neighbors.
+
+    metric : str
+        The metric used by the knn algorithm to calculate the distances.
+        It must be 'cosine', 'euclidean' or 'minkowski', otherwise this function does nothing.
+
+    C : int, default=100
+        Multiplier used to increase the tolerance of the acceptable precision differences.
+
+    p : int, default=None
+        This value is only used when metric is 'minkowski'.
+        A ValueError will be raised if metric is 'minkowski' and 'p' was not provided.
+
+    Returns
+    -------
+    fixed_scores : np.ndarray
+        An array of scores of shape ``(N,)`` for N examples with scores between 0 and 1.
+    """
+    if metric == "cosine":
+        tolerance = C * np.finfo(np.float_).epsneg
+    elif metric == "euclidean":
+        tolerance = np.sqrt(C * np.finfo(np.float_).eps)
+    elif metric == "minkowski":
+        if p is None:
+            raise ValueError("When metric is 'minkowski' you must specify the 'p' parameter")
+        tolerance = (C * np.finfo(np.float_).eps) ** (1 / p)
+    else:
+        return scores
 
-    # Set scores to 1 if the average distance is close to 0
-    inds = np.isclose(avg_distances, 0)
-    ood_features_scores[inds] = 1.0
-    return ood_features_scores
+    candidates_mask = avg_distances < tolerance
+    scores[candidates_mask] = 1
+    return scores
diff --git a/cleanlab/outlier.py b/cleanlab/outlier.py
--- a/cleanlab/outlier.py
+++ b/cleanlab/outlier.py
@@ -21,17 +21,19 @@
 """
 
 import warnings
+from typing import Dict, Optional, Tuple, Union
+
 import numpy as np
-from cleanlab.count import get_confident_thresholds
-from sklearn.neighbors import NearestNeighbors
 from sklearn.exceptions import NotFittedError
-from typing import Optional, Union, Tuple, Dict
+from sklearn.neighbors import NearestNeighbors
+
+from cleanlab.count import get_confident_thresholds
 from cleanlab.internal.label_quality_utils import (
     _subtract_confident_thresholds,
     get_normalized_entropy,
 )
 from cleanlab.internal.numerics import softmax
-from cleanlab.internal.outlier import transform_distances_to_scores
+from cleanlab.internal.outlier import correct_precision_errors, transform_distances_to_scores
 from cleanlab.internal.validation import assert_valid_inputs, labels_to_array
 from cleanlab.typing import LabelLike
 
@@ -470,6 +472,13 @@ def _get_ood_features_scores(
         ood_features_scores = transform_distances_to_scores(
             avg_knn_distances, t, scaling_factor=scaling_factor
         )
+        distance_metric = knn.metric
+        p = None
+        if distance_metric == "minkowski":
+            p = knn.p
+        ood_features_scores = correct_precision_errors(
+            ood_features_scores, avg_knn_distances, knn.metric, p=p
+        )
         return (ood_features_scores, knn)
 
 


Test patch:
diff --git a/tests/test_outlier.py b/tests/test_outlier.py
--- a/tests/test_outlier.py
+++ b/tests/test_outlier.py
@@ -14,19 +14,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with cleanlab.  If not, see <https://www.gnu.org/licenses/>.
 
-from hypothesis import example, settings, strategies as st
-from hypothesis import given
 import numpy as np
 import pandas as pd
 import pytest
-from cleanlab.benchmarking.noise_generation import generate_noise_matrix_from_trace
-from cleanlab.benchmarking.noise_generation import generate_noisy_labels
+from hypothesis import example, given, settings
+from hypothesis import strategies as st
+from sklearn.linear_model import LogisticRegression as LogReg
+from sklearn.neighbors import NearestNeighbors
+
 from cleanlab import count, outlier
+from cleanlab.benchmarking.noise_generation import (
+    generate_noise_matrix_from_trace,
+    generate_noisy_labels,
+)
 from cleanlab.count import get_confident_thresholds
-from cleanlab.outlier import OutOfDistribution
 from cleanlab.internal.label_quality_utils import get_normalized_entropy
-from sklearn.neighbors import NearestNeighbors
-from sklearn.linear_model import LogisticRegression as LogReg
+from cleanlab.outlier import OutOfDistribution
 
 
 def make_data(
@@ -660,7 +663,7 @@ def test_wrong_info_get_ood_predictions_scores():
 @given(
     fill_value=st.floats(
         min_value=5 * float(np.finfo(np.float_).eps),
-        max_value=10,
+        max_value=5,
         exclude_min=False,
         allow_subnormal=False,
         allow_infinity=False,
@@ -674,8 +677,42 @@ def test_scores_for_identical_examples(fill_value, K):
     N = 20
 
     features = np.full((N, K), fill_value=fill_value)
-    scores = OutOfDistribution().fit_score(features=features)
+    ood = OutOfDistribution()
+    scores = ood.fit_score(features=features, verbose=False)
+
+    # Dataset with only
+    expected_score = np.full(N, 1.0)
+    np.testing.assert_array_equal(
+        scores,
+        expected_score,
+        err_msg=f"The calculated distances were {ood.params['knn'].kneighbors()}",
+    )
+
+
+@given(K=st.integers(min_value=2, max_value=100))
+@settings(deadline=None)
+def test_scores_for_identical_examples_across_rows(K):
+    N = 20
+    fill_value = np.random.random(K)
+    features = np.full((N, K), fill_value=fill_value)
+    ood = OutOfDistribution()
+    scores = ood.fit_score(features=features, verbose=False)
 
     # Dataset with only
     expected_score = np.full(N, 1.0)
-    np.testing.assert_array_equal(scores, expected_score)
+    np.testing.assert_array_equal(
+        scores,
+        expected_score,
+        err_msg=f"The calculated distances were {ood.params['knn'].kneighbors()}",
+    )
+
+    if K < 4:
+        # This little changes should not affect euclidean calculation
+        features += np.random.random(features.shape) * np.sqrt(np.finfo(np.float_).eps)
+        ood = OutOfDistribution()
+        scores = ood.fit_score(features=features, verbose=False)
+        np.testing.assert_array_equal(
+            scores,
+            expected_score,
+            err_msg=f"The calculated distances were {ood.params['knn'].kneighbors()}",
+        )