Repository: HK3-Lab-Team/pytrousse

Issue:
DataFrameWithInfo features columns could be a subset of metadata columns
At the moment `DataFrameWithInfo` takes as parameters `metadata_cols` (tuple, i.e. the columns of the dataframe to be used as metadata) and `metadata_as_features` (bool, i.e. whether to consider all the metadata columns as features). It could be useful to set only a subset of the metadata columns as features, or even to exclude some columns both from the metadata and the features sets.

To accomplish these goals, `DataFrameWithInfo` could accept both `metadata_cols` and `feature_cols` parameters as tuples of strings enumerating the metadata and the feature columns, respectively. I suggest `feature_cols` to be an optional parameter, by default equal to `None`, meaning that all the columns but the metadata columns are features columns.

Discussion:
alessiamarcolini: @lorenz-gorini you could try to look at the PR commit-by-commit and not the entire diff at once

Patch:
diff --git a/scripts/use_anonymize_database.py b/scripts/use_anonymize_database.py
index 2a105d5..a49de15 100644
--- a/scripts/use_anonymize_database.py
+++ b/scripts/use_anonymize_database.py
@@ -12,5 +12,9 @@ output_data_dir = os.path.join(os.getcwd(), "data")
 df_sani = pd.read_csv(df_sani_dir)
 
 anonymize_data(
-    df_sani, "Sani_15300", private_cols_to_remove, private_cols_to_map, output_data_dir,
+    df_sani,
+    "Sani_15300",
+    private_cols_to_remove,
+    private_cols_to_map,
+    output_data_dir,
 )
diff --git a/scripts/use_dataframe_with_info.py b/scripts/use_dataframe_with_info.py
index 242457d..4425d65 100644
--- a/scripts/use_dataframe_with_info.py
+++ b/scripts/use_dataframe_with_info.py
@@ -4,7 +4,10 @@ import time
 from trousse.dataset import Dataset
 
 df_sani_dir = os.path.join(
-    "/home/lorenzo-hk3lab/WorkspaceHK3Lab/", "smvet", "data", "Sani_15300_anonym.csv",
+    "/home/lorenzo-hk3lab/WorkspaceHK3Lab/",
+    "smvet",
+    "data",
+    "Sani_15300_anonym.csv",
 )
 
 metadata_cols = (
diff --git a/src/trousse/dataset.py b/src/trousse/dataset.py
index a303b97..d19b051 100644
--- a/src/trousse/dataset.py
+++ b/src/trousse/dataset.py
@@ -214,10 +214,10 @@ class Dataset:
     def __init__(
         self,
         metadata_cols: Tuple = (),
+        feature_cols: Tuple = None,
         data_file: str = None,
         df_object: pd.DataFrame = None,
         nan_percentage_threshold: float = 0.999,
-        metadata_as_features: bool = False,
         new_columns_encoding_maps: Union[
             DefaultDict[str, List[FeatureOperation]], None
         ] = None,
@@ -232,9 +232,12 @@ class Dataset:
         ----------
         metadata_cols: Tuple[str], optional
             Tuple with the name of the columns that have metadata information related to
-            the sample. If ``metadata_as_features`` is set to False, these will not
-            be considered in following analysis or by "med_exam_col_list" property.
+            the sample.
             Default set to ()
+        feature_cols: Tuple[str], optional
+            Tuple with the name of the columns that contains sample features.
+            Default is None, meaning that all the columns but the ``metadata_cols`` will be
+            considered as features.
         data_file: str, optional
             Path to the csv file containing data. Either this or ``df_object`` must be
             provided. In case ``df_object`` is provided, this will not be considered.
@@ -246,10 +249,6 @@ class Dataset:
         nan_percentage_threshold: float, optional
             Float value in the range [0,1] describing the threshold of
             NaN values count to decide if the column is relevant or not.
-        metadata_as_features: bool
-            When set to ``True``, ``metadata_cols`` will be considered in following
-            analysis and among the features for preprocessing, otherwise they
-            will not.  Default set to False.
         new_columns_encoding_maps: Union[
             DefaultDict[str, List[FeatureOperation]], None
         ], optional
@@ -265,7 +264,11 @@ class Dataset:
         else:
             self.df = df_object
 
-        self.metadata_cols = set(metadata_cols)
+        self._metadata_cols = set(metadata_cols)
+        if feature_cols is None:
+            self._feature_cols = set(self.df.columns) - self.metadata_cols
+        else:
+            self._feature_cols = set(feature_cols)
         self.nan_percentage_threshold = nan_percentage_threshold
 
         # Dict of Lists ->
@@ -279,14 +282,32 @@ class Dataset:
         # Columns generated by Feature Refactoring (e.g.: encoding, bin_splitting)
         self.derived_columns = set()
 
-        # Option to choose whether to include "metadata_cols" in
-        # later analysis (treating them as normal features or not)
-        self.metadata_as_features = metadata_as_features
-
     # =====================
     # =    PROPERTIES     =
     # =====================
 
+    @property
+    def metadata_cols(self) -> Set[str]:
+        """Return columns representing metadata
+
+        Returns
+        -------
+        Set[str]
+            Metadata columns
+        """
+        return self._metadata_cols
+
+    @property
+    def feature_cols(self) -> Set[str]:
+        """Return columns representing features
+
+        Returns
+        -------
+        Set[str]
+            Feature columns
+        """
+        return self._feature_cols
+
     @property
     def many_nan_columns(self) -> Set[str]:
         """
@@ -303,7 +324,7 @@ class Dataset:
             ``nan_percentage_threshold`` attribute.
         """
         many_nan_columns = set()
-        for c in self.df.columns:
+        for c in self.feature_cols:
             # Check number of NaN
             if (
                 sum(self.df[c].isna())
@@ -324,7 +345,7 @@ class Dataset:
             Set of column names with only one repeated value
         """
         same_value_columns = set()
-        for c in self.df.columns:
+        for c in self.feature_cols:
             # Check number of unique values
             if len(self.df[c].unique()) == 1:
                 same_value_columns.add(c)
@@ -366,10 +387,8 @@ class Dataset:
 
         # TODO: Exclude NaN columns (self.nan_cols) from `col_list` too (so they will
         #  not be included in num_categorical_cols just for one not-Nan value)
-        if self.metadata_as_features:
-            col_list = set(self.df.columns) - same_value_cols
-        else:
-            col_list = set(self.df.columns) - same_value_cols - self.metadata_cols
+
+        col_list = self.feature_cols - same_value_cols
 
         (
             mixed_type_cols,
@@ -606,9 +625,9 @@ class Dataset:
         ----------
         col_list: Tuple[str], optional
             Tuple of the name of columns that should be considered.
-            If set to None, every column of ``df`` attribute will be considered.
+            If set to None, only the columns in ``self.feature_cols`` property.
         """
-        col_list = self.df.columns if col_list is None else col_list
+        col_list = self.feature_cols if col_list is None else col_list
         column_type_dict_list = Parallel(n_jobs=-1)(
             delayed(_find_single_column_type)(df_col=self.df[col]) for col in col_list
         )
@@ -664,7 +683,7 @@ class Dataset:
             # derived_columns is also derived by metadata_cols only and therefore
             # must be inserted in metadata_cols set, too
             if is_metadata_cols:
-                self.metadata_cols = self.metadata_cols.union(
+                self._metadata_cols = self.metadata_cols.union(
                     set(feature_operation.derived_columns)
                 )
             # Add the derived columns to the list of the instance
diff --git a/src/trousse/feature_fix.py b/src/trousse/feature_fix.py
index 475212a..3350363 100644
--- a/src/trousse/feature_fix.py
+++ b/src/trousse/feature_fix.py
@@ -7,8 +7,7 @@ import pandas as pd
 from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
 
 from .dataset import Dataset, FeatureOperation, copy_dataset_with_new_df
-from .feature_enum import (ENCODED_COLUMN_SUFFIX, EncodingFunctions,
-                           OperationTypeEnum)
+from .feature_enum import ENCODED_COLUMN_SUFFIX, EncodingFunctions, OperationTypeEnum
 
 logger = logging.getLogger(__name__)
 


Test patch:
diff --git a/tests/featureoperation_util.py b/tests/featureoperation_util.py
index e556f16..4042988 100644
--- a/tests/featureoperation_util.py
+++ b/tests/featureoperation_util.py
@@ -78,8 +78,16 @@ def eq_featureoperation_combs():
     ):
         # Unpack tuples from itertools
         op_type_1, op_type_2, is_op_type_equal = op_type_tuple
-        orig_column_1, orig_column_2, is_orig_column_equal, = orig_column_tuple
-        deriv_column_1, deriv_column_2, is_deriv_column_equal, = deriv_column_tuple
+        (
+            orig_column_1,
+            orig_column_2,
+            is_orig_column_equal,
+        ) = orig_column_tuple
+        (
+            deriv_column_1,
+            deriv_column_2,
+            is_deriv_column_equal,
+        ) = deriv_column_tuple
         encoder_1, encoder_2, is_encoder_equal = encoder_tuple
 
         is_equal_label = (
diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py
index cc136bb..192ae7a 100644
--- a/tests/integration/test_dataset.py
+++ b/tests/integration/test_dataset.py
@@ -160,16 +160,44 @@ class Describe_Dataset:
         assert categ_cols == expected_categ_cols
 
     @pytest.mark.parametrize(
-        "metadata_as_features, expected_column_list_type",
+        "feature_cols, expected_column_list_type",
         [
             (
-                True,
+                {"metadata_num_col"},
+                ColumnListByType(
+                    mixed_type_cols=set(),
+                    same_value_cols=set(),
+                    numerical_cols={
+                        "metadata_num_col",
+                    },
+                    med_exam_col_list={
+                        "metadata_num_col",
+                    },
+                    str_cols=set(),
+                    str_categorical_cols=set(),
+                    num_categorical_cols=set(),
+                    other_cols=set(),
+                    bool_cols=set(),
+                ),
+            ),
+            (
+                {
+                    "metadata_num_col",
+                    "mixed_type_col",
+                    "same_col",
+                    "numerical_col",
+                    "bool_col",
+                    "interval_col",
+                    "nan_col",
+                    "string_col",
+                    "str_categorical_col",
+                    "datetime_col",
+                },
                 ColumnListByType(
                     mixed_type_cols={"mixed_type_col"},
                     same_value_cols={"same_col"},
                     numerical_cols={
                         "numerical_col",
-                        "num_categorical_col",
                         "bool_col",
                         "interval_col",
                         "nan_col",
@@ -177,7 +205,6 @@ class Describe_Dataset:
                     },
                     med_exam_col_list={
                         "numerical_col",
-                        "num_categorical_col",
                         "bool_col",
                         "interval_col",
                         "nan_col",
@@ -185,13 +212,13 @@ class Describe_Dataset:
                     },
                     str_cols={"string_col", "str_categorical_col"},
                     str_categorical_cols={"str_categorical_col"},
-                    num_categorical_cols={"num_categorical_col", "nan_col"},
+                    num_categorical_cols={"nan_col"},
                     other_cols={"datetime_col"},
                     bool_cols={"bool_col"},
                 ),
             ),
             (
-                False,
+                None,
                 ColumnListByType(
                     mixed_type_cols={"mixed_type_col"},
                     same_value_cols={"same_col"},
@@ -218,12 +245,12 @@ class Describe_Dataset:
             ),
         ],
     )
-    def test_column_list_by_type(self, metadata_as_features, expected_column_list_type):
+    def test_column_list_by_type(self, feature_cols, expected_column_list_type):
         df_multi_type = DataFrameMock.df_multi_type(sample_size=200)
         dataset = Dataset(
             df_object=df_multi_type,
             metadata_cols=("metadata_num_col",),
-            metadata_as_features=metadata_as_features,
+            feature_cols=feature_cols,
         )
 
         col_list_by_type = dataset.column_list_by_type
@@ -232,10 +259,17 @@ class Describe_Dataset:
         assert col_list_by_type == expected_column_list_type
 
     @pytest.mark.parametrize(
-        "metadata_as_features, expected_med_exam_col_list",
+        "feature_cols, expected_med_exam_col_list",
         [
             (
-                True,
+                {
+                    "numerical_col",
+                    "num_categorical_col",
+                    "bool_col",
+                    "interval_col",
+                    "nan_col",
+                    "metadata_num_col",
+                },
                 {
                     "numerical_col",
                     "num_categorical_col",
@@ -246,7 +280,23 @@ class Describe_Dataset:
                 },
             ),
             (
-                False,
+                {
+                    "numerical_col",
+                    "num_categorical_col",
+                    "bool_col",
+                    "interval_col",
+                    "nan_col",
+                },
+                {
+                    "numerical_col",
+                    "num_categorical_col",
+                    "bool_col",
+                    "interval_col",
+                    "nan_col",
+                },
+            ),
+            (
+                None,
                 {
                     "numerical_col",
                     "num_categorical_col",
@@ -257,12 +307,12 @@ class Describe_Dataset:
             ),
         ],
     )
-    def test_med_exam_col_list(self, metadata_as_features, expected_med_exam_col_list):
+    def test_med_exam_col_list(self, feature_cols, expected_med_exam_col_list):
         df_multi_type = DataFrameMock.df_multi_type(sample_size=200)
         dataset = Dataset(
             df_object=df_multi_type,
             metadata_cols=("metadata_num_col",),
-            metadata_as_features=metadata_as_features,
+            feature_cols=feature_cols,
         )
 
         med_exam_col_list = dataset.med_exam_col_list
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py
new file mode 100644
index 0000000..b2bb497
--- /dev/null
+++ b/tests/unit/test_dataset.py
@@ -0,0 +1,69 @@
+import pytest
+
+from trousse.dataset import Dataset
+
+from ..dataset_util import DataFrameMock
+
+
+class DescribeDataset:
+    @pytest.mark.parametrize(
+        "metadata_cols",
+        [
+            (("metadata_num_col")),
+            (("metadata_num_col, string_col")),
+            (()),
+        ],
+    )
+    def it_knows_its_metadata_cols(self, metadata_cols):
+
+        df = DataFrameMock.df_multi_type(10)
+        dataset = Dataset(df_object=df, metadata_cols=metadata_cols)
+
+        metadata_cols_ = dataset.metadata_cols
+
+        assert type(metadata_cols_) == set
+        assert metadata_cols_ == set(metadata_cols)
+
+    @pytest.mark.parametrize(
+        "metadata_cols, feature_cols, expected_feature_cols",
+        [
+            (  # metadata and features non overlapping
+                ("metadata_num_col",),
+                ("str_categorical_col",),
+                {"str_categorical_col"},
+            ),
+            (  # metadata and features overlapping
+                ("metadata_num_col", "string_col"),
+                ("metadata_num_col", "str_categorical_col"),
+                {"metadata_num_col", "str_categorical_col"},
+            ),
+            (  # features None --> features are all columns but metadata
+                ("metadata_num_col",),
+                None,
+                {
+                    "numerical_col",
+                    "same_col",
+                    "bool_col",
+                    "mixed_type_col",
+                    "nan_col",
+                    "num_categorical_col",
+                    "string_col",
+                    "str_categorical_col",
+                    "interval_col",
+                    "datetime_col",
+                },
+            ),
+        ],
+    )
+    def it_knows_its_feature_cols(
+        self, metadata_cols, feature_cols, expected_feature_cols
+    ):
+        df = DataFrameMock.df_multi_type(10)
+        dataset = Dataset(
+            df_object=df, metadata_cols=metadata_cols, feature_cols=feature_cols
+        )
+
+        feature_cols_ = dataset.feature_cols
+
+        assert type(feature_cols_) == set
+        assert feature_cols_ == expected_feature_cols