Repository: mindsdb/lightwood

Issue:
STL decomposition
# Why

To improve the set of tools available for time series mixers to use if it is deemed useful.

# How

This PR does a _bunch_ of stuff, namely:

* Big refactor in the TS transform step. We now default to using pandas date/time utilities, meaning that indexes are now proper timestamps (multi-index in the case of grouped tasks). This both simplifies the logic and makes it more robust for things like inferring new rows according to the observed series frequencies.
* Introduces the `NeuralTs` mixer, which inherits from the vanilla `Neural` but adds time-series specific logic. This was done to keep specialized procedures separated from the (simpler) classification and regression tasks. JsonAI dispatch has been modified accordingly to make use of this.
* Available time series groups at training time are now logged and registered inside the `StatisticalAnalysis` object.
* Some simplifications to `ConcatedEncodedDs`, though we should probably remove this abstraction entirely (see #746).
* Moves a bunch of TS utilities to `lightwood.helpers.ts`
* `ts_analysis` phase now finds (and fits) optimal deseasonalizer and detrender objects for each series available at train time, for which we use Optuna.
* The respective methods that mixers can use to leverage these STL blocks are in `mixers.helpers.ts`
* `LightGBMArray`, `SkTime`, `Prophet` have been updated to use STL blocks, controllable by JsonAI args if needed. (NOTE: NeuralTs calls the blocks but there's no effect because it uses encoded values inside the DS, this will be addressed in a new PR)
* Encoder `TsNumericEncoder` has been simplified to avoid storing the sign, instead simply applying the underlying normalizer. This seems to be especially useful for `NeuralTs` to achieve improved forecasts, but formal benchmarks are still pending.
* Modifications to the `analysis.nc` block to work with all the above.


Discussion:

Resolving patch:
diff --git a/lightwood/mixer/helpers/ts.py b/lightwood/mixer/helpers/ts.py
--- a/lightwood/mixer/helpers/ts.py
+++ b/lightwood/mixer/helpers/ts.py
@@ -11,28 +11,32 @@
 from lightwood.helpers.ts import get_group_matches, get_ts_groups
 
 
-def _transform_target(ts_analysis: Dict[str, Dict], target_df: pd.DataFrame, group: tuple):
-    target_df = deepcopy(target_df)  # needed because methods like get_level_values actually have undisclosed side effects  # noqa
+def _transform_target(ts_analysis: Dict[str, Dict], df: pd.DataFrame, freq, group: tuple):
+    df = deepcopy(df)  # needed because methods like get_level_values actually have undisclosed side effects  # noqa
     transformer = ts_analysis['stl_transforms'][group]['transformer']
-    if isinstance(target_df.index, pd.MultiIndex) and len(target_df.index.levels) > 1:
-        return transformer.transform(target_df.droplevel(0).to_period())
-    elif isinstance(target_df.index, pd.MultiIndex):
-        temp_s = pd.Series(data=target_df.values)
-        temp_s.index = target_df.index.levels[0]  # warning: do not use get_level_values as it removes inferred freq
+    if isinstance(df.index, pd.MultiIndex) and len(df.index.levels) > 1:
+        temp_s = df.droplevel(0)
+        temp_s.index = pd.date_range(start=temp_s.index[0], freq=freq, periods=len(temp_s))
+        return transformer.transform(temp_s.to_period())
+    elif isinstance(df.index, pd.MultiIndex):
+        temp_s = pd.Series(data=df.values)
+        temp_s.index = df.index.levels[0]  # warning: do not use get_level_values as it removes inferred freq
         return transformer.transform(temp_s.to_period())
     else:
-        return transformer.transform(target_df.to_period())
+        return transformer.transform(df.to_period())
 
 
-def _inverse_transform_target(ts_analysis: Dict[str, Dict], predictions: pd.DataFrame, group: tuple):
+def _inverse_transform_target(ts_analysis: Dict[str, Dict], predictions: pd.DataFrame, freq, group: tuple):
     predictions = deepcopy(predictions)  # needed because methods like get_level_values actually have undisclosed side effects  # noqa
     transformer = ts_analysis['stl_transforms'][group]['transformer']
     if isinstance(predictions.index, pd.MultiIndex) and len(predictions.index.levels) > 1:
-        return transformer.inverse_transform(predictions.droplevel(0).to_period())
+        temp_s = predictions.droplevel(0)
+        temp_s.index = pd.date_range(start=temp_s.index[0], freq=freq, periods=len(temp_s))
+        return transformer.inverse_transform(temp_s.to_period())
     elif isinstance(predictions.index, pd.MultiIndex):
         temp_s = pd.Series(data=predictions)
         temp_s.index = predictions.index.levels[0]  # warning: do not use get_level_values as it removes inferred freq
-        return transformer.transform(temp_s.to_period())
+        return transformer.inverse_transform(temp_s.to_period())
     else:
         return transformer.inverse_transform(predictions.to_period())
 
@@ -61,22 +65,23 @@ def _apply_stl_on_training(
 
     for group in ts_analysis['group_combinations']:
         if len(ts_analysis['group_combinations']) == 1 or group != '__default':
+            freq = ts_analysis['sample_freqs'][group]
             train_idxs, train_subset = get_group_matches(train_data.data_frame, group, gby)
             dev_idxs, dev_subset = get_group_matches(dev_data.data_frame, group, gby)
 
-            train_data.data_frame[target].loc[train_idxs] = _transform_target(ts_analysis,
-                                                                              train_subset[target], group).values
-
-            dev_data.data_frame[target].loc[dev_idxs] = _transform_target(ts_analysis,
-                                                                          dev_subset[target], group).values
-
-            # shift all timestep cols here by respective offset
-            for timestep in range(1, tss.horizon):
-                train_data.data_frame[f'{target}_timestep_{timestep}'].loc[train_idxs] = \
-                    train_data.data_frame[target].loc[train_idxs].shift(-timestep)
+            if train_subset.size > 0:
+                train_data.data_frame[target].loc[train_idxs] = _transform_target(ts_analysis, train_subset[target],
+                                                                                  freq, group).values
+                for timestep in range(1, tss.horizon):
+                    train_data.data_frame[f'{target}_timestep_{timestep}'].loc[train_idxs] = \
+                        train_data.data_frame[target].loc[train_idxs].shift(-timestep)
 
-                dev_data.data_frame[f'{target}_timestep_{timestep}'].loc[dev_idxs] = \
-                    dev_data.data_frame[target].loc[dev_idxs].shift(-timestep)
+            if dev_subset.size > 0:
+                dev_data.data_frame[target].loc[dev_idxs] = _transform_target(ts_analysis, dev_subset[target],
+                                                                              freq, group).values
+                for timestep in range(1, tss.horizon):
+                    dev_data.data_frame[f'{target}_timestep_{timestep}'].loc[dev_idxs] = \
+                        dev_data.data_frame[target].loc[dev_idxs].shift(-timestep)
 
 
 def _stl_transform(
@@ -100,16 +105,16 @@ def _stl_transform(
     """  # noqa
     gby = tss.group_by if tss.group_by else []
     midx = pd.MultiIndex.from_frame(ds.data_frame.reset_index()[[*gby, 'index']])
-    midx.levels[0].freq = ds.data_frame['__mdb_inferred_freq'].iloc[0]
+    midx.levels[0].freq = ts_analysis['sample_freqs']['__default']  # note: assumes same freq across all groups
     ds.data_frame.index = midx
     ydf.index = midx
     groups = get_ts_groups(ds.data_frame, tss)
     for group in groups:
         group = group if group in ts_analysis['group_combinations'] else '__default'
         if len(ts_analysis['group_combinations']) == 1 or group != '__default':
+            freq = ts_analysis['sample_freqs'][group]
             idxs, subset = get_group_matches(ds.data_frame, group, gby)
-            ds.data_frame[target].loc[idxs] = _transform_target(
-                ts_analysis, subset[target], group).values
+            ds.data_frame[target].loc[idxs] = _transform_target(ts_analysis, subset[target], freq, group).values
     return ds.data_frame  # TODO: check that the side effects actually worked outside the fn scope and avoid returning?
 
 
@@ -133,10 +138,12 @@ def _stl_inverse_transform(
         group = group if group in ts_analysis['group_combinations'] else '__default'
         if len(ts_analysis['group_combinations']) == 1 or group != '__default':
             idxs, subset = get_group_matches(ds.data_frame, group, gby)
+            freq = ts_analysis['sample_freqs'][group]
             for timestep in range(tss.horizon):
                 ydf[f'prediction_{timestep}'].loc[idxs] = _inverse_transform_target(ts_analysis,
                                                                                     ydf[f'prediction_{timestep}'].loc[
                                                                                         idxs],
+                                                                                    freq,
                                                                                     group
                                                                                     ).values
     return ydf.reset_index(drop=True)


Test patch: