Repository: HK3-Lab-Team/pytrousse

Issue:
Dataset property: columns types 
- numerical_cols [2b]
- categorical_cols [2b]
- boolean_cols [2b]
- string_cols [2b]
- mixed_cols [2b]

Discussion:

Patch:
diff --git a/src/trousse/dataset.py b/src/trousse/dataset.py
index efa95d7..450f923 100644
--- a/src/trousse/dataset.py
+++ b/src/trousse/dataset.py
@@ -4,7 +4,7 @@ import dbm
 import logging
 import os
 import shelve
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import DefaultDict, Dict, List, Set, Tuple, Union
 
@@ -20,6 +20,7 @@ from .exceptions import (
 from .feature_enum import OperationTypeEnum
 from .feature_operation import FeatureOperation
 from .settings import CATEG_COL_THRESHOLD
+from .util import lazy_property
 
 logger = logging.getLogger(__name__)
 
@@ -52,31 +53,6 @@ _COL_NAME_COLUMN = "col_name"
 _COL_TYPE_COLUMN = "col_type"
 
 
-def _find_samples_by_type(df: pd.DataFrame, samples_type: str) -> Set[str]:
-    """
-    Return names of the sample that share the same value ``samples_type``
-
-    This function accepts a pandas DataFrame ``df`` with two columns: "col_name",
-    "col_type". Then it finds the rows with "col_type" value == ``samples_type`` and
-    returns a set with the "col_name" values of those rows.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-        Pandas DataFrame with 2 columns: "col_name" and "col_type"
-    samples_type: str
-        Value of "col_type" column of ``df``. The "col_name" of the samples with
-        this value will be included in the returned set
-
-    Returns
-    -------
-    set
-        Set with "col_name" value of the rows that have "col_type"
-        value == ``samples_type``
-    """
-    return set(df[df[_COL_TYPE_COLUMN] == samples_type][_COL_NAME_COLUMN].values)
-
-
 def _find_single_column_type(df_col: pd.Series) -> Dict[str, str]:
     """
     Analyze the ``df_col`` to find the type of its values.
@@ -125,74 +101,22 @@ def _find_single_column_type(df_col: pd.Series) -> Dict[str, str]:
         return {_COL_NAME_COLUMN: col, _COL_TYPE_COLUMN: "mixed_type_col"}
 
 
-def _split_columns_by_type_parallel(
-    df: pd.DataFrame, col_list: List[str]
-) -> Tuple[Set[str]]:
-    """
-    Find column types of DataFrame ``df``
-
-    For performance reasons, joblib is employed to analyze columns in parallel
-    processes.
-    For each column, after computing each value type, the function compares
-    them to check if they are different (in this case the column type is
-    "mixed_type_col"). If not, the based on the type of the column first element,
-    the function returns a column type as follows:
-    - float/int -> "numerical_col"
-    - bool -> "bool_col"
-    - str -> "string_col"
-    - other types -> "other_col"
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-        DataFrame containing columns to be analyzed
-    col_list: List[str]
-        List of the column names that will be analyzed
-
-    Returns
-    -------
-    set:
-        Set of columns containing values of different types
-    set:
-        Set of columns containing numerical values
-    set:
-        Set of columns containing string values
-    set:
-        Set of columns containing boolean values
-    set:
-        Set of the remaining columns that are in ``col_list`` but do not belong to any
-        other set
-    """
-    column_type_dict = Parallel(n_jobs=-1)(
-        delayed(_find_single_column_type)(df_col=df[col]) for col in col_list
-    )
-    column_type_df = pd.DataFrame(column_type_dict)
-
-    mixed_type_cols = _find_samples_by_type(column_type_df, "mixed_type_col")
-    numerical_cols = _find_samples_by_type(column_type_df, "numerical_col")
-    str_cols = _find_samples_by_type(column_type_df, "string_col")
-    other_cols = _find_samples_by_type(column_type_df, "other_col")
-    bool_cols = _find_samples_by_type(column_type_df, "bool_col")
-
-    return mixed_type_cols, numerical_cols, str_cols, bool_cols, other_cols
-
-
 @dataclass
-class ColumnListByType:
+class _ColumnListByType:
     """
     This dataclass is to gather the different column types inside a pd.DataFrame.
     The columns are split according to the type of their values.
     """
 
-    constant_cols: Set
-    mixed_type_cols: Set
-    numerical_cols: Set
-    med_exam_col_list: Set
-    str_cols: Set
-    str_categorical_cols: Set
-    num_categorical_cols: Set
-    other_cols: Set
-    bool_cols: Set
+    constant_cols: Set = field(default_factory=set)
+    mixed_type_cols: Set = field(default_factory=set)
+    numerical_cols: Set = field(default_factory=set)
+    med_exam_col_list: Set = field(default_factory=set)
+    str_cols: Set = field(default_factory=set)
+    str_categorical_cols: Set = field(default_factory=set)
+    num_categorical_cols: Set = field(default_factory=set)
+    other_cols: Set = field(default_factory=set)
+    bool_cols: Set = field(default_factory=set)
 
     def __str__(self):
         return (
@@ -363,8 +287,8 @@ class Dataset:
         """
         return self.many_nan_columns.union(self.constant_cols)
 
-    @property
-    def column_list_by_type(self) -> ColumnListByType:
+    @lazy_property
+    def _columns_type(self) -> _ColumnListByType:
         """
         Analyze the instance and return an object with the column list split by type.
 
@@ -375,8 +299,8 @@ class Dataset:
 
         Returns
         -------
-        ColumnListByType
-            ColumnListByType instance containing the column list split by type
+        _ColumnListByType
+            _ColumnListByType instance containing the column list split by type
         """
         constant_cols = self.constant_cols
 
@@ -385,13 +309,39 @@ class Dataset:
 
         col_list = self.feature_cols - constant_cols
 
-        (
-            mixed_type_cols,
-            numerical_cols,
-            str_cols,
-            bool_cols,
-            other_cols,
-        ) = _split_columns_by_type_parallel(self.df, col_list)
+        mixed_type_cols = set()
+        numerical_cols = set()
+        str_cols = set()
+        bool_cols = set()
+        other_cols = set()
+        categorical_cols = set()
+
+        PD_INFER_TYPE_MAP = {
+            "string": str_cols,
+            "bytes": other_cols,
+            "floating": numerical_cols,
+            "integer": numerical_cols,
+            "mixed-integer": mixed_type_cols,
+            "mixed-integer-float": numerical_cols,
+            "decimal": numerical_cols,
+            "complex": numerical_cols,
+            "boolean": bool_cols,
+            "datetime64": other_cols,
+            "datetime": other_cols,
+            "date": other_cols,
+            "timedelta64": other_cols,
+            "timedelta": other_cols,
+            "time": other_cols,
+            "period": other_cols,
+            "mixed": mixed_type_cols,
+            "interval": numerical_cols,
+            "category": categorical_cols,
+            "categorical": categorical_cols,
+        }
+
+        for col in col_list:
+            col_type = pd.api.types.infer_dtype(self.df[col], skipna=True)
+            PD_INFER_TYPE_MAP[col_type].add(col)
 
         str_categorical_cols = self._get_categorical_cols(str_cols)
         num_categorical_cols = self._get_categorical_cols(numerical_cols)
@@ -399,7 +349,15 @@ class Dataset:
             numerical_cols | bool_cols - constant_cols - self.metadata_cols
         )
 
-        return ColumnListByType(
+        for categorical_col in categorical_cols:
+            if categorical_col.dtype.categories.inferred_type == "integer":
+                num_categorical_cols.add(categorical_col)
+            elif categorical_col.dtype.categories.inferred_type == "string":
+                str_categorical_cols.add(categorical_col)
+            else:
+                raise RuntimeError("there is something wrong with the type guessing...")
+
+        return _ColumnListByType(
             mixed_type_cols=mixed_type_cols,
             constant_cols=constant_cols,
             numerical_cols=numerical_cols | bool_cols,  # TODO: Remove bool_cols
@@ -411,6 +369,103 @@ class Dataset:
             other_cols=other_cols,
         )
 
+    @property
+    def mixed_type_columns(self) -> Set[str]:
+        """Return the name of the columns with mixed type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with mixed type
+        """
+        return self._columns_type.mixed_type_cols
+
+    @property
+    def numerical_columns(self) -> Set[str]:
+        """Return the name of the columns with numerical type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with numerical type
+        """
+        return self._columns_type.numerical_cols
+
+    @property
+    def med_exam_col_list(self) -> Set[str]:
+        """
+        Get the name of the columns containing numerical values (metadata excluded).
+
+        The method will exclude from numerical columns the ones that have the same
+        repeated value, and the ones that contain metadata, but it will include columns
+        with many NaN
+
+        Returns
+        -------
+        Set
+            Set containing ``numerical_cols`` without ``metadata_cols`` and
+            ``constant_cols``
+        """
+        return self._columns_type.med_exam_col_list
+
+    @property
+    def str_columns(self) -> Set[str]:
+        """Return the name of the columns with string type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with string type
+        """
+        return self._columns_type.str_cols
+
+    @property
+    def str_categorical_columns(self) -> Set[str]:
+        """Return the name of the columns with string categorical type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with string categorical type
+        """
+        return self._columns_type.str_categorical_cols
+
+    @property
+    def num_categorical_columns(self) -> Set[str]:
+        """Return the name of the columns with numerical categorical type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with numerical categorical type
+        """
+        return self._columns_type.num_categorical_cols
+
+    @property
+    def bool_columns(self) -> Set[str]:
+        """Return the name of the columns with boolean type.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with boolean type
+        """
+        return self._columns_type.bool_cols
+
+    @property
+    def other_type_columns(self) -> Set[str]:
+        """Return the name of the columns with non-conventional type.
+
+        Types that are included in this category are: bytes, datetime64, datetime, date,
+        timedelta64, timedelta, time, period.
+
+        Returns
+        -------
+        Set[str]
+            The names of the columns with non-conventional type
+        """
+        return self._columns_type.other_cols
+
     def _get_categorical_cols(self, col_list: Tuple[str]) -> Set[str]:
         """
         Identify every categorical column in dataset.
@@ -465,7 +520,7 @@ class Dataset:
         Set[str]
             Set of columns with values of different types
         """
-        return self.column_list_by_type.mixed_type_cols
+        return self._columns_type.mixed_type_cols
 
     @property
     def to_be_encoded_cat_cols(self):
@@ -481,7 +536,7 @@ class Dataset:
 
         """
         to_be_encoded_categorical_cols = set()
-        cols_by_type = self.column_list_by_type
+        cols_by_type = self._columns_type
         # TODO: Check this because maybe categorical columns that are numerical, do
         #  not need encoding probably!
         categorical_cols = (
@@ -493,23 +548,6 @@ class Dataset:
 
         return to_be_encoded_categorical_cols
 
-    @property
-    def med_exam_col_list(self) -> Set:
-        """
-        Get the name of the columns containing numerical values (metadata excluded).
-
-        The method will exclude from numerical columns the ones that have the same
-        repeated value, and the ones that contain metadata, but it will include columns
-        with many NaN
-
-        Returns
-        -------
-        Set
-            Set containing ``numerical_cols`` without ``metadata_cols`` and
-            ``constant_cols``
-        """
-        return self.column_list_by_type.med_exam_col_list
-
     # =====================
     # =    METHODS        =
     # =====================
@@ -903,7 +941,7 @@ class Dataset:
             ``df`` attribute.
         """
         return (
-            f"{self.column_list_by_type}"
+            f"{self._columns_type}"
             f"\nColumns with many NaN: {len(self.many_nan_columns)}"
         )
 
diff --git a/src/trousse/util.py b/src/trousse/util.py
new file mode 100644
index 0000000..52dd817
--- /dev/null
+++ b/src/trousse/util.py
@@ -0,0 +1,6 @@
+import functools
+from typing import Any, Callable
+
+
+def lazy_property(f: Callable[..., Any]):
+    return property(functools.lru_cache()(f))


Test patch:
diff --git a/tests/dataset_util.py b/tests/dataset_util.py
index e57c23e..aed6e38 100644
--- a/tests/dataset_util.py
+++ b/tests/dataset_util.py
@@ -211,11 +211,9 @@ class DataFrameMock:
             "str_categorical_col": pd.Series(
                 ["category_0", "category_1", "category_2", "category_3", "category_4"]
                 * (sample_size // 5),
-                dtype="category",
             ),
             "num_categorical_col": pd.Series(
                 [0, 1, 2, 3, 4] * (sample_size // 5),
-                dtype="category",
             ),
             "numerical_col": [0.05 * i for i in range(sample_size)],
             "datetime_col": [date(2000 + i, 8, 1) for i in range(sample_size)],
diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py
index f2735ac..bfcb88a 100644
--- a/tests/integration/test_dataset.py
+++ b/tests/integration/test_dataset.py
@@ -7,12 +7,10 @@ import pandas as pd
 import pytest
 
 from trousse.dataset import (
-    ColumnListByType,
     Dataset,
     FeatureOperation,
-    _find_samples_by_type,
+    _ColumnListByType,
     _find_single_column_type,
-    _split_columns_by_type_parallel,
     copy_dataset_with_new_df,
     get_df_from_csv,
     read_file,
@@ -164,7 +162,7 @@ class Describe_Dataset:
         [
             (
                 {"metadata_num_col"},
-                ColumnListByType(
+                _ColumnListByType(
                     mixed_type_cols=set(),
                     constant_cols=set(),
                     numerical_cols={
@@ -193,7 +191,7 @@ class Describe_Dataset:
                     "str_categorical_col",
                     "datetime_col",
                 },
-                ColumnListByType(
+                _ColumnListByType(
                     mixed_type_cols={"mixed_type_col"},
                     constant_cols={"same_col"},
                     numerical_cols={
@@ -219,7 +217,7 @@ class Describe_Dataset:
             ),
             (
                 None,
-                ColumnListByType(
+                _ColumnListByType(
                     mixed_type_cols={"mixed_type_col"},
                     constant_cols={"same_col"},
                     numerical_cols={
@@ -253,9 +251,9 @@ class Describe_Dataset:
             feature_cols=feature_cols,
         )
 
-        col_list_by_type = dataset.column_list_by_type
+        col_list_by_type = dataset._columns_type
 
-        assert isinstance(col_list_by_type, ColumnListByType)
+        assert isinstance(col_list_by_type, _ColumnListByType)
         assert col_list_by_type == expected_column_list_type
 
     @pytest.mark.parametrize(
@@ -848,54 +846,6 @@ def test_find_single_column_type(request, series_type, expected_col_type_dict):
     assert col_type_dict == expected_col_type_dict
 
 
-@pytest.mark.parametrize(
-    "col_type, expected_column_single_type_set",
-    [
-        ("bool_col", {"bool_col_0", "bool_col_1"}),
-        ("string_col", {"string_col_0", "string_col_1", "string_col_2"}),
-        ("numerical_col", {"numerical_col_0"}),
-        ("other_col", {"other_col_0"}),
-        (
-            "mixed_type_col",
-            {
-                "mixed_type_col_0",
-                "mixed_type_col_1",
-                "mixed_type_col_2",
-                "mixed_type_col_3",
-            },
-        ),
-    ],
-)
-def test_find_columns_by_type(request, col_type, expected_column_single_type_set):
-    df_col_names_by_type = DataFrameMock.df_column_names_by_type()
-
-    column_single_type_set = _find_samples_by_type(df_col_names_by_type, col_type)
-
-    assert column_single_type_set == expected_column_single_type_set
-
-
-def test_split_columns_by_type_parallel(request):
-    df_by_type = DataFrameMock.df_multi_type(sample_size=10)
-    col_list = df_by_type.columns
-
-    cols_by_type_tuple = _split_columns_by_type_parallel(df_by_type, col_list)
-
-    assert cols_by_type_tuple == (
-        {"mixed_type_col"},
-        {
-            "numerical_col",
-            "interval_col",
-            "num_categorical_col",
-            "nan_col",
-            "metadata_num_col",
-            "same_col",
-        },
-        {"string_col", "str_categorical_col"},
-        {"bool_col"},
-        {"datetime_col"},
-    )
-
-
 def test_copy_dataset_with_new_df(dataset_with_operations):
     new_df = DataFrameMock.df_generic(10)
 
diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py
index b2bb497..5b175db 100644
--- a/tests/unit/test_dataset.py
+++ b/tests/unit/test_dataset.py
@@ -1,8 +1,9 @@
 import pytest
 
-from trousse.dataset import Dataset
+from trousse.dataset import Dataset, _ColumnListByType
 
 from ..dataset_util import DataFrameMock
+from ..unitutil import initializer_mock, property_mock
 
 
 class DescribeDataset:
@@ -15,7 +16,6 @@ class DescribeDataset:
         ],
     )
     def it_knows_its_metadata_cols(self, metadata_cols):
-
         df = DataFrameMock.df_multi_type(10)
         dataset = Dataset(df_object=df, metadata_cols=metadata_cols)
 
@@ -67,3 +67,113 @@ class DescribeDataset:
 
         assert type(feature_cols_) == set
         assert feature_cols_ == expected_feature_cols
+
+    def it_knows_its_mixed_type_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(mixed_type_cols={"mixed0", "mixed1"})
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        mixed_type_columns_ = dataset.mixed_type_columns
+
+        assert type(mixed_type_columns_) == set
+        assert mixed_type_columns_ == {"mixed0", "mixed1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_numerical_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(
+            numerical_cols={"numerical0", "numerical1"}
+        )
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        numerical_columns_ = dataset.numerical_columns
+
+        assert type(numerical_columns_) == set
+        assert numerical_columns_ == {"numerical0", "numerical1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_med_exam_col_list(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(med_exam_col_list={"med0", "med1"})
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        med_exam_col_list_ = dataset.med_exam_col_list
+
+        assert type(med_exam_col_list_) == set
+        assert med_exam_col_list_ == {"med0", "med1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_str_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(str_cols={"str0", "str1"})
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        str_columns_ = dataset.str_columns
+
+        assert type(str_columns_) == set
+        assert str_columns_ == {"str0", "str1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_str_categorical_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(
+            str_categorical_cols={"strcat0", "strcat1"}
+        )
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        str_categorical_columns_ = dataset.str_categorical_columns
+
+        assert type(str_categorical_columns_) == set
+        assert str_categorical_columns_ == {"strcat0", "strcat1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_num_categorical_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(
+            num_categorical_cols={"numcat0", "numcat1"}
+        )
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        num_categorical_columns_ = dataset.num_categorical_columns
+
+        assert type(num_categorical_columns_) == set
+        assert num_categorical_columns_ == {"numcat0", "numcat1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_bool_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(bool_cols={"bool0", "bool1"})
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        bool_columns_ = dataset.bool_columns
+
+        assert type(bool_columns_) == set
+        assert bool_columns_ == {"bool0", "bool1"}
+        _columns_type.assert_called_once()
+
+    def it_knows_its_other_type_columns(self, request):
+        _columns_type = property_mock(request, Dataset, "_columns_type")
+        column_list_by_type = _ColumnListByType(other_cols={"other0", "other1"})
+        _columns_type.return_value = column_list_by_type
+        initializer_mock(request, Dataset)
+        dataset = Dataset(data_file="fake/path")
+
+        other_type_columns_ = dataset.other_type_columns
+
+        assert type(other_type_columns_) == set
+        assert other_type_columns_ == {"other0", "other1"}
+        _columns_type.assert_called_once()