Repository: mindsdb/lightwood

Issue:
STL decomposition
# Why

To improve the set of tools available for time series mixers to use if it is deemed useful.

# How

This PR does a _bunch_ of stuff, namely:

* Big refactor in the TS transform step. We now default to using pandas date/time utilities, meaning that indexes are now proper timestamps (multi-index in the case of grouped tasks). This both simplifies the logic and makes it more robust for things like inferring new rows according to the observed series frequencies.
* Introduces the `NeuralTs` mixer, which inherits from the vanilla `Neural` but adds time-series specific logic. This was done to keep specialized procedures separated from the (simpler) classification and regression tasks. JsonAI dispatch has been modified accordingly to make use of this.
* Available time series groups at training time are now logged and registered inside the `StatisticalAnalysis` object.
* Some simplifications to `ConcatedEncodedDs`, though we should probably remove this abstraction entirely (see #746).
* Moves a bunch of TS utilities to `lightwood.helpers.ts`
* `ts_analysis` phase now finds (and fits) optimal deseasonalizer and detrender objects for each series available at train time, for which we use Optuna.
* The respective methods that mixers can use to leverage these STL blocks are in `mixers.helpers.ts`
* `LightGBMArray`, `SkTime`, `Prophet` have been updated to use STL blocks, controllable by JsonAI args if needed. (NOTE: NeuralTs calls the blocks but there's no effect because it uses encoded values inside the DS, this will be addressed in a new PR)
* Encoder `TsNumericEncoder` has been simplified to avoid storing the sign, instead simply applying the underlying normalizer. This seems to be especially useful for `NeuralTs` to achieve improved forecasts, but formal benchmarks are still pending.
* Modifications to the `analysis.nc` block to work with all the above.


Discussion:

Resolving patch:
diff --git a/lightwood/mixer/lightgbm_array.py b/lightwood/mixer/lightgbm_array.py
--- a/lightwood/mixer/lightgbm_array.py
+++ b/lightwood/mixer/lightgbm_array.py
@@ -38,31 +38,33 @@ def __init__(
         self.horizon = tss.horizon
         self.submodel_stop_after = stop_after / self.horizon
         self.target = target
+        self.offset_pred_cols = [f'{self.target}_timestep_{i}' for i in range(1, self.horizon)]
+        if set(input_cols) != set(self.tss.order_by):
+            input_cols.remove(*self.tss.order_by)
+        for col in self.offset_pred_cols:
+            dtype_dict[col] = dtype_dict[self.target]
         self.models = [LightGBM(self.submodel_stop_after,
-                                target,
+                                target_col,
                                 dtype_dict,
                                 input_cols,
-                                fit_on_dev,
-                                False,  # use_optuna
+                                False,  # fit_on_dev,
+                                True,  # use_optuna
                                 target_encoder)
-                       for _ in range(self.horizon)]
+                       for _, target_col in zip(range(self.horizon), [target] + self.offset_pred_cols)]
         self.ts_analysis = ts_analysis
         self.supports_proba = False
-        self.use_stl_blocks = True
+        self.use_stl = False
         self.stable = True
 
     def _fit(self, train_data: EncodedDs, dev_data: EncodedDs, submodel_method='fit') -> None:
         original_train = deepcopy(train_data.data_frame)
         original_dev = deepcopy(dev_data.data_frame)
 
-        if self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             _apply_stl_on_training(train_data, dev_data, self.target, self.tss, self.ts_analysis)
 
         for timestep in range(self.horizon):
-            if timestep > 0:
-                train_data.data_frame[self.target] = train_data.data_frame[f'{self.target}_timestep_{timestep}']
-                dev_data.data_frame[self.target] = dev_data.data_frame[f'{self.target}_timestep_{timestep}']
-            getattr(self.models[timestep], submodel_method)(train_data, dev_data)  # call submodel_method to fit
+            getattr(self.models[timestep], submodel_method)(train_data, dev_data)
 
         # restore dfs
         train_data.data_frame = original_train
@@ -87,13 +89,13 @@ def __call__(self, ds: Union[EncodedDs, ConcatedEncodedDs],
                            index=np.arange(length),
                            columns=[f'prediction_{i}' for i in range(self.horizon)])
 
-        if self.use_stl_blocks and self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             ds.data_frame = _stl_transform(ydf, ds, self.target, self.tss, self.ts_analysis)
 
         for timestep in range(self.horizon):
             ydf[f'prediction_{timestep}'] = self.models[timestep](ds, args)['prediction'].values
 
-        if self.use_stl_blocks and self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             ydf = _stl_inverse_transform(ydf, ds, self.tss, self.ts_analysis)
 
         if self.models[0].positive_domain:
diff --git a/lightwood/mixer/neural_ts.py b/lightwood/mixer/neural_ts.py
--- a/lightwood/mixer/neural_ts.py
+++ b/lightwood/mixer/neural_ts.py
@@ -36,6 +36,7 @@ def __init__(
             search_hyperparameters: bool,
             ts_analysis: Dict[str, Dict],
             n_epochs: Optional[int] = None,
+            use_stl: Optional[bool] = False
     ):
         """
         Subclassed Neural mixer used for time series forecasting scenarios. 
@@ -65,6 +66,7 @@ def __init__(
         self.ts_analysis = ts_analysis
         self.net_class = DefaultNet if net == 'DefaultNet' else ArNet
         self.stable = True
+        self.use_stl = use_stl
 
     def _select_criterion(self) -> torch.nn.Module:
         if self.dtype_dict[self.target] in (dtype.integer, dtype.float, dtype.num_tsarray, dtype.quantity):
@@ -88,7 +90,7 @@ def _fit(self, train_data: EncodedDs, dev_data: EncodedDs) -> None:
         original_dev = deepcopy(dev_data.data_frame)
 
         # Use STL blocks if available
-        if self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             _apply_stl_on_training(train_data, dev_data, self.target, self.timeseries_settings, self.ts_analysis)
 
         # ConcatedEncodedDs
@@ -145,7 +147,7 @@ def __call__(self, ds: EncodedDs,
                            dtype=object,
                            columns=pred_cols)
 
-        if self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             ds.data_frame = _stl_transform(ydf, ds, self.target, self.timeseries_settings, self.ts_analysis)
 
         with torch.no_grad():
@@ -171,7 +173,7 @@ def __call__(self, ds: EncodedDs,
             decoded_predictions = np.expand_dims(decoded_predictions, axis=1)
         ydf[pred_cols] = decoded_predictions
 
-        if self.ts_analysis.get('stl_transforms', False):
+        if self.use_stl and self.ts_analysis.get('stl_transforms', False):
             ydf = _stl_inverse_transform(ydf, ds, self.timeseries_settings, self.ts_analysis)
 
         ydf['prediction'] = ydf.values.tolist()
diff --git a/lightwood/mixer/sktime.py b/lightwood/mixer/sktime.py
--- a/lightwood/mixer/sktime.py
+++ b/lightwood/mixer/sktime.py
@@ -38,7 +38,7 @@ def __init__(
             model_path: str = 'statsforecast.StatsForecastAutoARIMA',
             auto_size: bool = True,
             hyperparam_search: bool = False,
-            use_decomposers: bool = True
+            use_stl: bool = False
     ):
         """
         This mixer is a wrapper around the popular time series library sktime. It exhibits different behavior compared
@@ -64,7 +64,7 @@ def __init__(
         :param model_path: sktime forecaster to use as underlying model(s). Should be a string with format "$module.$class' where '$module' is inside `sktime.forecasting`. Default is 'arima.AutoARIMA'.
         :param hyperparam_search: bool that indicates whether to perform the hyperparameter tuning or not.
         :param auto_size: whether to filter out old data points if training split is bigger than a certain threshold (defined by the dataset sampling frequency). Enabled by default to avoid long training times in big datasets.
-        :param use_decomposers: Whether to use de-trenders and de-seasonalizers fitted in the timeseries analysis phase. Enabled by default.
+        :param use_stl: Whether to use de-trenders and de-seasonalizers fitted in the timeseries analysis phase.
         """  # noqa
         super().__init__(stop_after)
         self.stable = True
@@ -78,7 +78,7 @@ def __init__(
         self.grouped_by = ['__default'] if not ts_analysis['tss'].group_by else ts_analysis['tss'].group_by
         self.auto_size = auto_size
         self.cutoff_factor = 4  # times the detected maximum seasonal period
-        self.use_decomposers = use_decomposers
+        self.use_stl = use_stl
 
         # optuna hyperparameter tuning
         self.models = {}
@@ -151,7 +151,7 @@ def _fit(self, data):
 
             model_pipeline = [("forecaster", model_class(**kwargs))]
 
-            if self.use_decomposers:
+            if self.use_stl:
                 model_pipeline.insert(0, ("detrender",
                                           self.ts_analysis['stl_transforms'][group]["transformer"].detrender))
                 model_pipeline.insert(0, ("deseasonalizer",


Test patch: