Repository: HK3-Lab-Team/pytrousse

Issue:
Dataset method: nan_columns
nan_columns(tolerance) [2c] 

Tolerance is a (optional, default=1) float number (0 to 1) representing the ratio “nan samples”/”total samples” for the column to be considered a “nan column”.


Discussion:
lorenz-gorini: It needs `rebase` after the other merged PR, I think

Patch:
diff --git a/src/trousse/dataset.py b/src/trousse/dataset.py
index 450f923..ce949a1 100644
--- a/src/trousse/dataset.py
+++ b/src/trousse/dataset.py
@@ -141,7 +141,6 @@ class Dataset:
         feature_cols: Tuple = None,
         data_file: str = None,
         df_object: pd.DataFrame = None,
-        nan_percentage_threshold: float = 0.999,
         new_columns_encoding_maps: Union[
             DefaultDict[str, List[FeatureOperation]], None
         ] = None,
@@ -170,9 +169,6 @@ class Dataset:
             Pandas DataFrame instance containing the data. Either this or data_file
             must be provided. In case ``data_file`` is provided, only this will
             be considered as data. Default set to None.
-        nan_percentage_threshold: float, optional
-            Float value in the range [0,1] describing the threshold of
-            NaN values count to decide if the column is relevant or not.
         new_columns_encoding_maps: Union[
             DefaultDict[str, List[FeatureOperation]], None
         ], optional
@@ -193,7 +189,6 @@ class Dataset:
             self._feature_cols = set(self.df.columns) - self.metadata_cols
         else:
             self._feature_cols = set(feature_cols)
-        self.nan_percentage_threshold = nan_percentage_threshold
 
         # Dict of Lists ->
         #         key: column_name,
@@ -232,31 +227,31 @@ class Dataset:
         """
         return self._feature_cols
 
-    @property
-    def many_nan_columns(self) -> Set[str]:
-        """
-        Return name of the columns containing many NaN.
+    def nan_columns(self, nan_ratio: float = 1) -> Set[str]:
+        """Return name of the columns containing at least a ``nan_ratio`` ratio of NaNs.
+
+        Select the columns where the nan_ratio of NaN values over the
+        sample count is higher than ``nan_ratio`` (in range [0,1]).
 
-        This property selects the columns where the ratio of NaN values over the
-        sample count is higher than ``nan_percentage_threshold`` attribute
-        (in range [0,1]).
+        Parameters
+        ----------
+        nan_ratio : float, optional
+            Minimum ratio “nan samples”/”total samples” for the column to be considered
+            a “nan column”. Default is 1, meaning that only the columns entirely composed
+            by NaNs will be returned.
 
         Returns
         -------
         Set[str]
-            Set of column names with NaN ratio higher than
-            ``nan_percentage_threshold`` attribute.
+            Set of column names with NaN ratio higher than ``nan_ratio`` parameter.
         """
-        many_nan_columns = set()
+        nan_columns = set()
         for c in self.feature_cols:
             # Check number of NaN
-            if (
-                sum(self.df[c].isna())
-                > self.nan_percentage_threshold * self.df.shape[0]
-            ):
-                many_nan_columns.add(c)
+            if sum(self.df[c].isna()) > nan_ratio * self.df.shape[0]:
+                nan_columns.add(c)
 
-        return many_nan_columns
+        return nan_columns
 
     @property
     def constant_cols(self) -> Set[str]:
@@ -276,8 +271,8 @@ class Dataset:
         """
         Return name of the columns containing many NaN or only one repeated value.
 
-        This function return the name of the column that were included in
-        ``constant_cols`` or ``many_nan_cols`` attributes
+        This function return the name of the column that were returned by
+        ``constant_cols`` property or ``nan_columns`` method.
 
         Returns
         -------
@@ -285,7 +280,7 @@ class Dataset:
             Set containing the name of the columns with many NaNs or with only
             one repeated value
         """
-        return self.many_nan_columns.union(self.constant_cols)
+        return self.nan_columns(nan_ratio=0.999).union(self.constant_cols)
 
     @lazy_property
     def _columns_type(self) -> _ColumnListByType:
@@ -552,30 +547,6 @@ class Dataset:
     # =    METHODS        =
     # =====================
 
-    def least_nan_cols(self, threshold: int) -> Set:
-        """
-        Get the features with a count of NaN values lower than the ``threshold`` argument.
-
-        Parameters
-        ----------
-        threshold: int
-            Number of samples that have a NaN value. If the count of NaN values
-            from a column is lower than ``threshold`` value, the name of the column
-            will be in the returned set
-
-        Returns
-        -------
-        Set
-            Set of column names with a NaN count lower than the ``threshold`` argument
-        """
-        best_feature_list = set()
-        # TODO: Add argument col_list to select the columns to analyze
-        for c in self.med_exam_col_list:
-            if sum(self.df[c].isna()) < threshold:
-                best_feature_list.add(c)
-
-        return best_feature_list
-
     def get_encoded_string_values_map(
         self, column_name: str
     ) -> Union[Dict[int, str], None]:
@@ -942,7 +913,7 @@ class Dataset:
         """
         return (
             f"{self._columns_type}"
-            f"\nColumns with many NaN: {len(self.many_nan_columns)}"
+            f"\nColumns with many NaN: {len(self.nan_columns(0.999))}"
         )
 
     def __call__(self) -> pd.DataFrame:


Test patch:
diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py
index bfcb88a..6b4111d 100644
--- a/tests/integration/test_dataset.py
+++ b/tests/integration/test_dataset.py
@@ -25,7 +25,7 @@ from ..fixtures import CSV
 
 class Describe_Dataset:
     @pytest.mark.parametrize(
-        "nan_ratio, n_columns, expected_many_nan_columns",
+        "nan_ratio, n_columns, expected_nan_columns",
         [
             (0.8, 2, {"nan_0", "nan_1"}),
             (0.8, 1, {"nan_0"}),
@@ -46,17 +46,15 @@ class Describe_Dataset:
             (1.0, 2, {"nan_0", "nan_1"}),
         ],
     )
-    def test_many_nan_columns(
-        self, request, nan_ratio, n_columns, expected_many_nan_columns
-    ):
+    def test_nan_columns(self, request, nan_ratio, n_columns, expected_nan_columns):
         df = DataFrameMock.df_many_nans(nan_ratio, n_columns)
-        dataset = Dataset(df_object=df, nan_percentage_threshold=nan_ratio - 0.01)
+        dataset = Dataset(df_object=df)
 
-        many_nan_columns = dataset.many_nan_columns
+        nan_columns = dataset.nan_columns(nan_ratio - 0.01)
 
-        assert len(many_nan_columns) == len(expected_many_nan_columns)
-        assert isinstance(many_nan_columns, set)
-        assert many_nan_columns == expected_many_nan_columns
+        assert len(nan_columns) == len(expected_nan_columns)
+        assert isinstance(nan_columns, set)
+        assert nan_columns == expected_nan_columns
 
     @pytest.mark.parametrize(
         "n_columns, expected_constant_columns",
@@ -318,24 +316,6 @@ class Describe_Dataset:
         assert isinstance(med_exam_col_list, set)
         assert med_exam_col_list == expected_med_exam_col_list
 
-    @pytest.mark.parametrize(
-        "nan_threshold, expected_least_nan_cols",
-        [
-            (10, {"0nan_col"}),
-            (101, {"0nan_col", "50nan_col"}),
-            (199, {"0nan_col", "50nan_col"}),
-            (200, {"0nan_col", "50nan_col", "99nan_col"}),
-        ],
-    )
-    def test_least_nan_cols(self, request, nan_threshold, expected_least_nan_cols):
-        df_multi_type = DataFrameMock.df_multi_nan_ratio(sample_size=200)
-        dataset = Dataset(df_object=df_multi_type)
-
-        least_nan_cols = dataset.least_nan_cols(nan_threshold)
-
-        assert isinstance(least_nan_cols, set)
-        assert least_nan_cols == expected_least_nan_cols
-
     @pytest.mark.parametrize(
         "duplicated_cols_count, expected_contains_dupl_cols_bool",
         [(0, False), (4, True), (2, True)],
@@ -778,6 +758,23 @@ class Describe_Dataset:
         assert type(columns_name)
         assert columns_name == expected_columns_name
 
+    def test_str(self):
+        df = DataFrameMock.df_multi_type(10)
+        dataset = Dataset(df_object=df)
+        expected_str = (
+            "Columns with:\n\t1.\tMixed types: "
+            "\t\t1\n\t2.\tNumerical types (float/int): \t6\n\t3.\tString types: "
+            "\t\t2\n\t4.\tBool types: \t\t1\n\t5.\tOther types: \t\t1\nAmong these "
+            "categories:\n\t1.\tString categorical columns: 1\n\t2.\tNumeric categorical"
+            " columns: 2\n\t3.\tMedical Exam columns (numerical, no metadata): 6\n\t4."
+            "\tOne repeated value: 1\nColumns with many NaN: 0"
+        )
+
+        str_ = str(dataset)
+
+        assert type(str_) == str
+        assert expected_str == str_
+
 
 class Describe_FeatureOperation:
     @pytest.mark.parametrize(
diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py
index 5b175db..246d51e 100644
--- a/tests/unit/test_dataset.py
+++ b/tests/unit/test_dataset.py
@@ -3,7 +3,7 @@ import pytest
 from trousse.dataset import Dataset, _ColumnListByType
 
 from ..dataset_util import DataFrameMock
-from ..unitutil import initializer_mock, property_mock
+from ..unitutil import initializer_mock, method_mock, property_mock
 
 
 class DescribeDataset:
@@ -177,3 +177,66 @@ class DescribeDataset:
         assert type(other_type_columns_) == set
         assert other_type_columns_ == {"other0", "other1"}
         _columns_type.assert_called_once()
+
+    def it_knows_its_str(self, request):
+        column_list_by_type = _ColumnListByType(
+            mixed_type_cols={"mixed0", "mixed1"},
+            constant_cols={"constant"},
+            numerical_cols={"numerical0", "numerical1"},
+            med_exam_col_list={"med0", "med1", "med2"},
+            str_cols={"str0", "str1"},
+            str_categorical_cols={"strcat0", "strcat1"},
+            num_categorical_cols={"numcat0", "numcat1"},
+            bool_cols={"bool0", "bool1"},
+            other_cols={"other0", "other1"},
+        )
+        expected_str = (
+            "Columns with:\n\t1.\tMixed types: \t\t2\n\t2.\tNumerical types"
+            " (float/int): \t2\n\t3.\tString types: \t\t2\n\t4.\tBool types: \t\t2\n\t5."
+            "\tOther types: \t\t2\nAmong these categories:\n\t1.\tString categorical "
+            "columns: 2\n\t2.\tNumeric categorical columns: 2\n\t3.\tMedical Exam columns "
+            "(numerical, no metadata): 3\n\t4.\tOne repeated value: 1"
+        )
+
+        str_ = str(column_list_by_type)
+
+        assert type(str_) == str
+        assert str_ == expected_str
+
+
+class DescribeColumnListByType:
+    def it_knows_its_str(self, request):
+        column_list_by_type_str = (
+            "Columns with:\n\t1.\tMixed types: \t\t2\n\t2.\tNumerical types"
+            " (float/int): \t2\n\t3.\tString types: \t\t2\n\t4.\tBool types: \t\t2\n\t5."
+            "\tOther types: \t\t2\nAmong these categories:\n\t1.\tString categorical "
+            "columns: 2\n\t2.\tNumeric categorical columns: 2\n\t3.\tMedical Exam columns "
+            "(numerical, no metadata): 3\n\t4.\tOne repeated value: 1"
+        )
+        _column_list_by_type_str = method_mock(request, _ColumnListByType, "__str__")
+        _column_list_by_type_str.return_value = column_list_by_type_str
+        _column_list_by_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(
+            mixed_type_cols={"mixed0", "mixed1"},
+            constant_cols={"constant"},
+            numerical_cols={"numerical0", "numerical1"},
+            med_exam_col_list={"med0", "med1", "med2"},
+            str_cols={"str0", "str1"},
+            str_categorical_cols={"strcat0", "strcat1"},
+            num_categorical_cols={"numcat0", "numcat1"},
+            bool_cols={"bool0", "bool1"},
+            other_cols={"other0", "other1"},
+        )
+        _column_list_by_type.return_value = column_list_by_type
+        _nan_columns = method_mock(request, Dataset, "nan_columns")
+        _nan_columns.return_value = {"nan0", "nan1"}
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+        expected_str = column_list_by_type_str + "\nColumns with many NaN: 2"
+
+        str_ = str(dataset)
+
+        assert type(str_) == str
+        assert str_ == expected_str
+        _column_list_by_type.assert_called_once
+        _nan_columns.assert_called_once_with(dataset, 0.999)