Repository: cleanlab/cleanlab

Issue:
Datalab issue type for null/missing feature values
[New Datalab issue type](https://docs.cleanlab.ai/master/cleanlab/datalab/guide/custom_issue_manager.html) called something like `null` that checks `features` for rows that are entirely missing / null values (across all columns).

Those rows should get flagged as `is_null_issue`. 
The quality score for each row can be the fraction of `features` which are missing in that row.

Make sure this issue check does not waste compute time if it is irrelevant, ie. first check that there even exist any missing values in the `features` at all before proceeding further. 


Discussion:
@jwmueller can you assign this issue to me?

Resolving patch:
diff --git a/cleanlab/datalab/internal/issue_manager/null.py b/cleanlab/datalab/internal/issue_manager/null.py
new file mode 100644
--- /dev/null
+++ b/cleanlab/datalab/internal/issue_manager/null.py
@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, List
+
+import numpy as np
+import pandas as pd
+from numpy import ndarray
+
+from cleanlab.datalab.internal.issue_manager import IssueManager
+
+if TYPE_CHECKING:  # pragma: no cover
+    import numpy.typing as npt
+
+
+class NullIssueManager(IssueManager):
+    """Manages issues related to null/missing values in the rows of features.
+
+    Parameters
+    ----------
+    datalab :
+        The Datalab instance that this issue manager searches for issues in.
+    """
+
+    description: ClassVar[
+        str
+    ] = """Whether the dataset has any missing/null values
+        """
+    issue_name: ClassVar[str] = "null"
+    verbosity_levels = {
+        0: ["average_null_score"],
+        1: ["most_common_issue"],
+        2: [],
+    }
+
+    @staticmethod
+    def _calculate_null_issues(features: npt.NDArray) -> tuple[ndarray, ndarray, Any]:
+        """Tracks the number of null values in each row of a feature array,
+        computes quality scores based on the fraction of null values in each row,
+        and returns a boolean array indicating whether each row only has null values."""
+        rows = features.shape[0]
+        cols = features.shape[1]
+        scores = np.ones(rows).astype(np.float32)
+        is_null_issue = np.full(rows, False)
+        null_tracker = np.isnan(features)
+        if null_tracker.any():
+            for row in range(rows):
+                if null_tracker[row].any():
+                    non_null_col_count = np.count_nonzero(~null_tracker[row])
+                    scores[row] = non_null_col_count / cols
+                    if scores[row] == 0.00:
+                        is_null_issue[row] = True
+        return is_null_issue, scores, null_tracker
+
+    def find_issues(
+        self,
+        features: Optional[npt.NDArray] = None,
+        **kwargs,
+    ) -> None:
+        if features is None:
+            raise ValueError("features must be provided to check for null values.")
+        is_null_issue, scores, null_tracker = self._calculate_null_issues(features=features)
+
+        self.issues = pd.DataFrame(
+            {
+                f"is_{self.issue_name}_issue": is_null_issue,
+                self.issue_score_key: scores,
+            },
+        )
+
+        self.summary = self.make_summary(score=scores.mean())
+        self.info = self.collect_info(null_tracker)
+
+    @staticmethod
+    def _most_common_issue(
+        null_tracker: np.ndarray,
+    ) -> dict[str, dict[str, str | int | list[int] | list[int | None]]]:
+        """
+        Identify and return the most common null value pattern across all rows
+        and count the number of rows with this pattern.
+
+        Parameters
+        ------------
+        null_tracker : np.ndarray
+            A boolean array of the same shape as features, where True indicates null/missing entries.
+
+        Returns
+        --------
+        Dict[str, Any]
+            A dictionary containing the most common issue pattern and the count of rows with this pattern.
+        """
+        # Convert the boolean null_tracker matrix into a list of strings.
+        most_frequent_pattern = "no_null"
+        rows_affected: List[int] = []
+        occurrence_of_most_frequent_pattern = 0
+        if null_tracker.any():
+            null_patterns_as_strings = [
+                "".join(map(str, row.astype(int).tolist())) for row in null_tracker if row.any()
+            ]
+
+            # Use Counter to efficiently count occurrences and find the most common pattern.
+            pattern_counter = Counter(null_patterns_as_strings)
+            (
+                most_frequent_pattern,
+                occurrence_of_most_frequent_pattern,
+            ) = pattern_counter.most_common(1)[0]
+            rows_affected = []
+            for idx, row in enumerate(null_patterns_as_strings):
+                if row == most_frequent_pattern:
+                    rows_affected.append(idx)
+        return {
+            "most_common_issue": {
+                "pattern": most_frequent_pattern,
+                "rows_affected": rows_affected,
+                "count": occurrence_of_most_frequent_pattern,
+            }
+        }
+
+    @staticmethod
+    def _column_impact(null_tracker: np.ndarray) -> Dict[str, List[float]]:
+        """
+        Calculate and return the impact of null values per column, represented as the proportion
+        of rows having null values in each column.
+
+        Parameters
+        ----------
+        null_tracker : np.ndarray
+            A boolean array of the same shape as features, where True indicates null/missing entries.
+
+        Returns
+        -------
+        Dict[str, List[float]]
+            A dictionary containing the impact per column, with values being a list
+            where each element is the percentage of rows having null values in the corresponding column.
+        """
+        # Calculate proportion of nulls in each column
+        proportion_of_nulls_per_column = null_tracker.mean(axis=0)
+
+        # Return result as a dictionary containing a list of proportions
+        return {"column_impact": proportion_of_nulls_per_column.tolist()}
+
+    def collect_info(self, null_tracker: np.ndarray) -> dict:
+        most_common_issue = self._most_common_issue(null_tracker=null_tracker)
+        column_impact = self._column_impact(null_tracker=null_tracker)
+        average_null_score = {"average_null_score": self.issues[self.issue_score_key].mean()}
+        issues_dict = {**average_null_score, **most_common_issue, **column_impact}
+        info_dict: Dict[str, Any] = {**issues_dict}
+        return info_dict


Test patch:
diff --git a/tests/datalab/issue_manager/test_null.py b/tests/datalab/issue_manager/test_null.py
new file mode 100644
--- /dev/null
+++ b/tests/datalab/issue_manager/test_null.py
@@ -0,0 +1,178 @@
+import numpy as np
+import pytest
+from hypothesis.extra.numpy import arrays, array_shapes
+from hypothesis.strategies import floats, just
+from hypothesis import HealthCheck, given, settings
+
+from cleanlab.datalab.internal.issue_manager.null import NullIssueManager
+
+SEED = 42
+
+
+class TestNullIssueManager:
+    @pytest.fixture
+    def embeddings(self):
+        np.random.seed(SEED)
+        embeddings_array = np.random.random((4, 3))
+        return embeddings_array
+
+    @pytest.fixture
+    def embeddings_with_null(self):
+        np.random.seed(SEED)
+        embeddings_array = np.random.random((4, 3))
+        embeddings_array[0][0] = np.NaN
+        embeddings_array[1] = np.NaN
+        return embeddings_array
+
+    @pytest.fixture
+    def issue_manager(self, lab):
+        return NullIssueManager(datalab=lab)
+
+    def test_init(self, lab, issue_manager):
+        assert issue_manager.datalab == lab
+
+    def test_find_issues(self, issue_manager, embeddings):
+        np.random.seed(SEED)
+        issue_manager.find_issues(features=embeddings)
+        issues_sort, summary_sort, info_sort = (
+            issue_manager.issues,
+            issue_manager.summary,
+            issue_manager.info,
+        )
+        expected_sorted_issue_mask = np.array([False, False, False, False])
+        assert np.all(
+            issues_sort["is_null_issue"] == expected_sorted_issue_mask
+        ), "Issue mask should be correct"
+        assert summary_sort["issue_type"][0] == "null"
+        assert summary_sort["score"][0] == pytest.approx(expected=1.0, abs=1e-7)
+        assert (
+            info_sort.get("average_null_score", None) is not None
+        ), "Should have average null score"
+        assert summary_sort["score"][0] == pytest.approx(
+            expected=info_sort["average_null_score"], abs=1e-7
+        )
+
+    def test_find_issues_with_null(self, issue_manager, embeddings_with_null):
+        np.random.seed(SEED)
+        issue_manager.find_issues(features=embeddings_with_null)
+        issues_sort, summary_sort, info_sort = (
+            issue_manager.issues,
+            issue_manager.summary,
+            issue_manager.info,
+        )
+        expected_sorted_issue_mask = np.array([False, True, False, False])
+        assert np.all(
+            issues_sort["is_null_issue"] == expected_sorted_issue_mask
+        ), "Issue mask should be correct"
+        assert summary_sort["issue_type"][0] == "null"
+        assert summary_sort["score"][0] == pytest.approx(expected=8 / 12, abs=1e-7)
+        assert (
+            info_sort.get("average_null_score", None) is not None
+        ), "Should have average null score"
+        assert summary_sort["score"][0] == pytest.approx(
+            expected=info_sort["average_null_score"], abs=1e-7
+        )
+
+    def test_report(self, issue_manager, embeddings):
+        np.random.seed(SEED)
+        issue_manager.find_issues(features=embeddings)
+        report = issue_manager.report(
+            issues=issue_manager.issues,
+            summary=issue_manager.summary,
+            info=issue_manager.info,
+        )
+
+        assert isinstance(report, str)
+        assert (
+            "----------------------- null issues ------------------------\n\n"
+            "Number of examples with this issue:"
+        ) in report
+
+        report = issue_manager.report(
+            issues=issue_manager.issues,
+            summary=issue_manager.summary,
+            info=issue_manager.info,
+            verbosity=3,
+        )
+        assert "Additional Information: " in report
+
+    def test_report_with_null(self, issue_manager, embeddings_with_null):
+        np.random.seed(SEED)
+        issue_manager.find_issues(features=embeddings_with_null)
+        report = issue_manager.report(
+            issues=issue_manager.issues,
+            summary=issue_manager.summary,
+            info=issue_manager.info,
+        )
+
+        assert isinstance(report, str)
+        assert (
+            "----------------------- null issues ------------------------\n\n"
+            "Number of examples with this issue:"
+        ) in report
+
+        report = issue_manager.report(
+            issues=issue_manager.issues,
+            summary=issue_manager.summary,
+            info=issue_manager.info,
+            verbosity=3,
+        )
+        assert "Additional Information: " in report
+
+    def test_collect_info(self, issue_manager, embeddings):
+        """Test some values in the info dict."""
+        issue_manager.find_issues(features=embeddings)
+        info = issue_manager.info
+        assert info["average_null_score"] == 1.0
+        assert info["most_common_issue"]["pattern"] == "no_null"
+        assert info["most_common_issue"]["count"] == 0
+        assert info["most_common_issue"]["rows_affected"] == []
+        assert info["column_impact"] == [0, 0, 0]
+
+    def test_collect_info_with_nulls(self, issue_manager, embeddings_with_null):
+        """Test some values in the info dict."""
+        issue_manager.find_issues(features=embeddings_with_null)
+        info = issue_manager.info
+        assert info["average_null_score"] == pytest.approx(expected=8 / 12, abs=1e-7)
+        assert info["most_common_issue"]["pattern"] == "100"
+        assert info["most_common_issue"]["count"] == 1
+        assert info["most_common_issue"]["rows_affected"] == [0]
+        assert info["column_impact"] == [0.5, 0.25, 0.25]
+
+    # Strategy for generating NaN values
+    nan_strategy = just(np.nan)
+
+    # Strategy for generating regular float values, including NaNs
+    float_with_nan = floats(allow_nan=True)
+
+    # Strategy for generating NumPy arrays with some NaN values
+    features_with_nan_strategy = arrays(
+        dtype=np.float64,
+        shape=array_shapes(min_dims=2, max_dims=2, min_side=1, max_side=5),
+        elements=float_with_nan,
+        fill=nan_strategy,
+    )
+
+    @settings(
+        suppress_health_check=[HealthCheck.function_scoped_fixture]
+    )  # No need to reset state of issue_manager fixture
+    @given(embeddings=features_with_nan_strategy)
+    def test_quality_scores_and_full_null_row_identification(self, issue_manager, embeddings):
+        # Run the find_issues method
+        issue_manager.find_issues(features=embeddings)
+        issues_sort, _, _ = (
+            issue_manager.issues,
+            issue_manager.summary,
+            issue_manager.info,
+        )
+
+        # Check for the two main properties:
+
+        # 1. The quality score for each row should be the fraction of features which are not null in that row.
+        non_null_fractions = [np.count_nonzero(~np.isnan(row)) / len(row) for row in embeddings]
+        scores = issues_sort[issue_manager.issue_score_key]
+        assert np.allclose(scores, non_null_fractions, atol=1e-7)
+
+        # 2. The rows that are marked as is_null_issue should ONLY be those rows which are 100% null values.
+        all_rows_are_null = np.all(np.isnan(embeddings), axis=1)
+        assert np.all(issues_sort["is_null_issue"] == all_rows_are_null)