Repository: KIT-IAI/pyWATTS

Issue:
Feature selection module
**What is your feature request related to?**
* Module request

**Please describe the module/problem**

A module to select features from a given feature space. Remove redundant features, keep features with high information value.

Possible approaches: filter, wrapper, hybrid
Possible selection methods: threshold, k-best


**Describe the solution you'd like**
Parameters: selection method, k
Input: Time series of candidate features
Output: Time series of selected features, weights of all candidate features

**Possible alternatives**
[sklearn's feature selection](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.feature_selection)

**What type of Feature Request**
* Module request


Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 30be6e8..8edb432 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,8 @@
 
 ### Changed
 * Retraining is triggered after all steps are transformed ([#117](https://github.com/KIT-IAI/pyWATTS/issues/117))
+* The sklean-wrapper includes now methods based on the SelectorMixin ([#185](https://github.com/KIT-IAI/pyWATTS/issues/185))
+
 
 ### Fixed
 * Try to handle non matching shapes in metrics. If not possible raise an exception.  ([#154](https://github.com/KIT-IAI/pyWATTS/issues/154))
diff --git a/examples/example.py b/examples/example.py
index 2a6c1bc..eb2ce99 100644
--- a/examples/example.py
+++ b/examples/example.py
@@ -7,13 +7,14 @@
 import pandas as pd
 from sklearn.linear_model import LinearRegression
 from sklearn.preprocessing import StandardScaler
+from sklearn.feature_selection import SelectKBest, f_regression
 
 # From pyWATTS the pipeline is imported
 from pywatts.callbacks import LinePlotCallback
 from pywatts.core.computation_mode import ComputationMode
 from pywatts.core.pipeline import Pipeline
 # All modules required for the pipeline are imported
-from pywatts.modules import CalendarExtraction, CalendarFeature, ClockShift, LinearInterpolater, SKLearnWrapper
+from pywatts.modules import CalendarExtraction, CalendarFeature, ClockShift, LinearInterpolater, SKLearnWrapper, Sampler
 from pywatts.summaries import RMSE
 
 # The main function is where the pipeline is created and run
@@ -21,7 +22,7 @@ if __name__ == "__main__":
     # Create a pipeline
     pipeline = Pipeline(path="../results")
 
-    # Extract dummy calender features, using holidays from Germany
+    # Extract dummy calendar features, using holidays from Germany
     # NOTE: CalendarExtraction can't return multiple features.
     calendar = CalendarExtraction(continent="Europe", country="Germany", features=[CalendarFeature.month,
                                                                                    CalendarFeature.weekday,
@@ -43,28 +44,38 @@ if __name__ == "__main__":
     shift_power_statistics2 = ClockShift(lag=2, name="ClockShift_Lag2"
                                          )(x=scale_power_statistics)
 
+    target_multiple_output = Sampler(24, name="sampled_data")(x=scale_power_statistics)
+
+    # Select features based on F-statistic
+    selected_features = SKLearnWrapper(
+        module=SelectKBest(score_func=f_regression, k=2)
+    )(
+        power_lag1=shift_power_statistics,
+        power_lag2=shift_power_statistics2,
+        calendar=calendar,
+        target=scale_power_statistics,
+    )
+
     # Create a linear regression that uses the lagged values to predict the current value
     # NOTE: SKLearnWrapper has to collect all **kwargs itself and fit it against target.
     #       It is also possible to implement a join/collect class
     regressor_power_statistics = SKLearnWrapper(
         module=LinearRegression(fit_intercept=True)
     )(
-        power_lag1=shift_power_statistics,
-        power_lag2=shift_power_statistics2,
-        calendar=calendar,
-        target=scale_power_statistics,
-        callbacks=[LinePlotCallback('linear_regression')],
+        features=selected_features,
+        target=target_multiple_output,
+        callbacks=[LinePlotCallback("linear_regression")],
     )
 
     # Rescale the predictions to be on the original time scale
     inverse_power_scale = power_scaler(
         x=regressor_power_statistics, computation_mode=ComputationMode.Transform,
-        use_inverse_transform=True, callbacks=[LinePlotCallback('rescale')]
+        use_inverse_transform=True, callbacks=[LinePlotCallback("rescale")]
     )
 
     # Calculate the root mean squared error (RMSE) between the linear regression and the true values
     # save it as csv file
-    rmse = RMSE()(y_hat=inverse_power_scale, y=pipeline["load_power_statistics"])
+    rmse = RMSE()(y_hat=inverse_power_scale, y=target_multiple_output)
 
     # Now, the pipeline is complete so we can run it and explore the results
     # Start the pipeline
diff --git a/pywatts/modules/wrappers/sklearn_wrapper.py b/pywatts/modules/wrappers/sklearn_wrapper.py
index 67a4be9..978552b 100644
--- a/pywatts/modules/wrappers/sklearn_wrapper.py
+++ b/pywatts/modules/wrappers/sklearn_wrapper.py
@@ -1,10 +1,11 @@
-import pickle
+import cloudpickle
 from typing import List, Tuple
 
 import numpy as np
 import sklearn
 import xarray as xr
 from sklearn.base import TransformerMixin
+from sklearn.feature_selection import SelectorMixin
 
 from pywatts.core.exceptions.kind_of_transform_does_not_exist_exception import KindOfTransformDoesNotExistException, \
     KindOfTransform
@@ -61,8 +62,9 @@ class SKLearnWrapper(BaseWrapper):
         self.targets = list(targets.keys())
         x = self._dataset_to_sklearn_input(inputs)
         target = self._dataset_to_sklearn_input(targets)
-        self.targets = list(
-            zip(targets.keys(), map(lambda x: x.shape[-1] if len(x.shape) > 1 else 1, list(targets.values()))))
+        self.targets = list(zip(targets.keys(),
+                                map(lambda t: t.values.reshape((t.shape[0], -1)).shape[-1] if len(t.shape) > 1 else 1,
+                                    list(targets.values()))))
         self.fit_result = self.module.fit(x, target)
         self.is_fitted = True
 
@@ -82,13 +84,13 @@ class SKLearnWrapper(BaseWrapper):
     def _sklearn_output_to_dataset(kwargs: xr.DataArray, prediction, targets: List[Tuple[str, int]]):
         reference = kwargs[list(kwargs)[0]]
         time_index = reference.indexes[_get_time_indexes(reference)[0]]
-        if len(targets) == 0:
+        if len(targets) == 0:  # sklearn module is a transformer
             coords = (
                 # first dimension is number of batches. We assume that this is the time.
                 ("time", time_index.values),
                 *[(f"dim_{j}", list(range(size))) for j, size in enumerate(prediction.shape[1:])])
             result = xr.DataArray(prediction, coords=coords)
-        else:
+        else:  # sklearn module is an estimator
             result = {}
             position = 0
             prediction = prediction.reshape(len(list(reference.coords.values())[0]), -1)
@@ -105,8 +107,12 @@ class SKLearnWrapper(BaseWrapper):
         :return: the transformed output
         """
         x_np = self._dataset_to_sklearn_input(kwargs)
+        target_names = self.targets
 
-        if isinstance(self.module, TransformerMixin):
+        if isinstance(self.module, SelectorMixin):
+            prediction = self.module.transform(x_np)
+            target_names = []  # output of selector must not match the shape of the target
+        elif isinstance(self.module, TransformerMixin):
             prediction = self.module.transform(x_np)
         elif "predict" in dir(self.module):
             prediction = self.module.predict(x_np)
@@ -115,7 +121,7 @@ class SKLearnWrapper(BaseWrapper):
                 f"The sklearn-module in {self.name} does not have a predict or transform method",
                 KindOfTransform.PREDICT_TRANSFORM)
 
-        return self._sklearn_output_to_dataset(kwargs, prediction, self.targets)
+        return self._sklearn_output_to_dataset(kwargs, prediction, target_names)
 
     def inverse_transform(self, **kwargs: xr.DataArray) -> xr.DataArray:
         """
@@ -124,14 +130,18 @@ class SKLearnWrapper(BaseWrapper):
         :return: the transformed output
         """
         x_np = self._dataset_to_sklearn_input(kwargs)
+        target_names = self.targets
+
         if self.has_inverse_transform:
             prediction = self.module.inverse_transform(x_np)
+            if isinstance(self.module, SelectorMixin):
+                target_names = []  # output of selector must not match the shape of the target
         else:
             raise KindOfTransformDoesNotExistException(
                 f"The sklearn-module in {self.name} does not have a inverse transform method",
                 KindOfTransform.INVERSE_TRANSFORM)
 
-        return self._sklearn_output_to_dataset(kwargs, prediction, self.targets)
+        return self._sklearn_output_to_dataset(kwargs, prediction, target_names)
 
     def predict_proba(self, **kwargs) -> xr.DataArray:
         """
@@ -150,12 +160,14 @@ class SKLearnWrapper(BaseWrapper):
         return self._sklearn_output_to_dataset(kwargs, prediction, self.targets)
 
     def save(self, fm: FileManager):
-        json = super().save(fm)
+        json_module = super().save(fm)
+        json_module["targets"] = self.targets
         file_path = fm.get_path(f'{self.name}.pickle')
         with open(file_path, 'wb') as outfile:
-            pickle.dump(obj=self.module, file=outfile)
-        json.update({"sklearn_module": file_path})
-        return json
+            cloudpickle.dump(obj=self.module, file=outfile)
+        json_module.update({"sklearn_module": file_path})
+        json_module.pop("params")
+        return json_module
 
     @classmethod
     def load(cls, load_information) -> 'SKLearnWrapper':
@@ -172,7 +184,8 @@ class SKLearnWrapper(BaseWrapper):
         """
         name = load_information["name"]
         with open(load_information["sklearn_module"], 'rb') as pickle_file:
-            module = pickle.load(pickle_file)
+            module = cloudpickle.load(pickle_file)
         module = cls(module=module, name=name)
         module.is_fitted = load_information["is_fitted"]
+        module.targets = load_information["targets"]
         return module


Test patch:
diff --git a/tests/unit/core/test_pipeline.py b/tests/unit/core/test_pipeline.py
index 96063dc..14fde1c 100644
--- a/tests/unit/core/test_pipeline.py
+++ b/tests/unit/core/test_pipeline.py
@@ -21,18 +21,14 @@ pipeline_json = {'id': 1,
                               'is_fitted': False,
                               'module': 'pywatts.modules.wrappers.sklearn_wrapper',
                               'name': 'StandardScaler',
-                              'params': {'copy': True, 'with_mean': True, 'with_std': True},
-                              'sklearn_module': os.path.join('test_pipeline', 'StandardScaler.pickle')},
+                              'sklearn_module': os.path.join('test_pipeline', 'StandardScaler.pickle'),
+                              'targets': []},
                              {'class': 'SKLearnWrapper',
                               'is_fitted': False,
                               'module': 'pywatts.modules.wrappers.sklearn_wrapper',
                               'name': 'LinearRegression',
-                              'params': {'copy_X': True,
-                                         'fit_intercept': True,
-                                         'n_jobs': None,
-                                         'positive': False,
-                                         'normalize': 'deprecated'},
-                              'sklearn_module': os.path.join('test_pipeline', 'LinearRegression.pickle')}],
+                              'sklearn_module': os.path.join('test_pipeline', 'LinearRegression.pickle'),
+                              'targets': []}],
                  'steps': [{'class': 'StartStep',
                             'default_run_setting': {'computation_mode': 4},
                             'id': 1,
@@ -147,7 +143,7 @@ class TestPipeline(unittest.TestCase):
         assert kwargs["obj"]["steps"] == pipeline_json["steps"]
 
     @patch('pywatts.core.pipeline.FileManager')
-    @patch('pywatts.modules.sklearn_wrapper.pickle')
+    @patch('pywatts.modules.sklearn_wrapper.cloudpickle')
     @patch('pywatts.core.pipeline.json')
     @patch("builtins.open", new_callable=mock_open)
     @patch('pywatts.core.pipeline.os.path.isdir')
diff --git a/tests/unit/wrapper/test_sklearn_wrapper.py b/tests/unit/wrapper/test_sklearn_wrapper.py
index 7b62a42..14a32ed 100644
--- a/tests/unit/wrapper/test_sklearn_wrapper.py
+++ b/tests/unit/wrapper/test_sklearn_wrapper.py
@@ -7,6 +7,7 @@ from sklearn.cluster import KMeans
 from sklearn.linear_model import LinearRegression
 from sklearn.mixture import GaussianMixture
 from sklearn.multioutput import MultiOutputRegressor
+from sklearn.feature_selection import SelectKBest, f_regression
 
 from sklearn.preprocessing import StandardScaler
 from sklearn.svm import SVC, SVR
@@ -33,18 +34,31 @@ class TestSklearnWrapper(unittest.TestCase):
         wrapper = SKLearnWrapper(module=scaler)
         self.assertFalse("mean_" in scaler.__dir__())
 
-        wrapper.fit(test = xr.DataArray([1, 2, 3, 4, 5]))
+        wrapper.fit(test=xr.DataArray([1, 2, 3, 4, 5]))
 
         self.assertTrue("mean_" in scaler.__dir__())
         self.assertIsNotNone(scaler.mean_)
 
+    def test_transform_TransformerMixin(self):
+        scaler = StandardScaler()
+        wrapper = SKLearnWrapper(module=scaler)
+        self.assertFalse("mean_" in scaler.__dir__())
+        time = pd.date_range('2000-01-08', freq='24H', periods=5)
+        test = xr.DataArray([2, 2, 2, 2, 2], dims=["time"], coords={'time': time})
+
+        wrapper.fit(test=test)
+        result = wrapper.transform(test=test)
+
+        self.assertListEqual(list(result), [0, 0, 0, 0, 0])
+        self.assertEqual(result.shape, (5, 1))
+
     def test_fit_RegressorMixin(self):
         lin_reg = LinearRegression()
         wrapper = SKLearnWrapper(module=lin_reg)
         self.assertFalse("coef_" in lin_reg.__dir__())
 
-        wrapper.fit(test = xr.DataArray([1, 2, 3, 4, 5]),
-                    target= xr.DataArray([2, 2, 2, 2, 2]))
+        wrapper.fit(test=xr.DataArray([1, 2, 3, 4, 5]),
+                    target=xr.DataArray([2, 2, 2, 2, 2]))
 
         self.assertTrue("coef_" in lin_reg.__dir__())
         self.assertIsNotNone(lin_reg.coef_)
@@ -55,13 +69,42 @@ class TestSklearnWrapper(unittest.TestCase):
         time = pd.date_range('2000-01-08', freq='24H', periods=1)
         bar = xr.DataArray([1], dims=["time"], coords={'time': time})
 
-        wrapper.fit(test = xr.DataArray([1, 2, 3, 4, 5]),
-                    target= xr.DataArray([2, 2, 2, 2, 2]))
-
+        wrapper.fit(test=xr.DataArray([1, 2, 3, 4, 5]),
+                    target=xr.DataArray([2, 2, 2, 2, 2]))
 
         result = wrapper.transform(bar=bar)
         assert result["target"].values[0] == 2.0
-        self.assertEqual(result["target"].shape, (1,1))
+        self.assertEqual(result["target"].shape, (1, 1))
+
+    def test_fit_SelectorMixin(self):
+        kbest = SelectKBest(score_func=f_regression, k=1)
+        wrapper = SKLearnWrapper(module=kbest)
+
+        eps = 0.001
+        wrapper.fit(feature1=xr.DataArray([x + eps for x in [2, 2, 3, 4, 4]]),
+                    feature2=xr.DataArray([1, 2, 3, 4, 5]),
+                    target=xr.DataArray([2, 2, 3, 4, 4]))
+
+        self.assertTrue("scores_" in kbest.__dir__())
+        self.assertIsNotNone(kbest.scores_)
+
+    def test_transform_SelectorMixin(self):
+        kbest = SelectKBest(score_func=f_regression, k=1)
+        wrapper = SKLearnWrapper(module=kbest)
+        eps = 0.001
+        time = pd.date_range('2000-01-08', freq='24H', periods=5)
+        target = xr.DataArray([2, 2, 3, 4, 4], dims=["time"], coords={'time': time})
+        feature1 = target + eps
+        feature2 = xr.DataArray([4, 4, 3, 2, 2], dims=["time"], coords={'time': time}) + eps
+
+        wrapper.fit(feature1=feature1,
+                    feature2=feature2,
+                    target=target)
+
+        result = wrapper.transform(feature1=feature1, feature2=feature2)
+
+        self.assertListEqual(list(result), list(feature2))
+        self.assertEqual(result.shape, (5, 1))
 
     def test_DensityMixin(self):
         gauss_density = GaussianMixture(n_components=2)
@@ -159,10 +202,8 @@ class TestSklearnWrapper(unittest.TestCase):
 
         wrapper.fit(bar=bar, target1=target, target2=target2)
 
-
         result = wrapper.transform(bar=foo)
         self.assertAlmostEqual(result["target1"].values[0], 2.0)
         self.assertAlmostEqual(result["target2"].values[0], 3.0)
         self.assertEqual(result["target1"].shape, (1, 1))
         self.assertEqual(result["target2"].shape, (1, 1))
-