Repository: mindsdb/lightwood

Issue:
Improve time series frequency inference
Right now we only do units of time according to the closest match. Which is fine for use cases with `1<X>` sampling rates with `X` some time unit like seconds, minutes, etc.

Predictions are incorrect if the forecasting task has a different multiplier, e.g. 3 minutes, 15 seconds, 2 hours. We should improve the underlying module to support these cases.


Discussion:

Resolving patch:
diff --git a/lightwood/helpers/ts.py b/lightwood/helpers/ts.py
--- a/lightwood/helpers/ts.py
+++ b/lightwood/helpers/ts.py
@@ -240,10 +240,13 @@ def detect_freq_period(deltas: pd.DataFrame, tss, n_points) -> tuple:
 
     diffs = [(tag, abs(deltas - secs)) for tag, secs in secs_to_interval.items()]
     freq, min_diff = sorted(diffs, key=lambda x: x[1])[0]
-    return freq_to_pandas(freq), freq_to_period.get(freq, 1)
+    multiplier = 1
+    if secs_to_interval[freq]:
+        multiplier += int(min_diff / secs_to_interval[freq])
+    return freq_to_pandas(freq, multiplier=multiplier), freq_to_period.get(freq, 1)
 
 
-def freq_to_pandas(freq, sample_row=None):
+def freq_to_pandas(freq, multiplier=1, sample_row=None):
     mapping = {
         'constant': 'N',
         'nanosecond': 'N',
@@ -262,7 +265,10 @@ def freq_to_pandas(freq, sample_row=None):
 
     # TODO: implement custom dispatch for better precision, use row sample if available:
     #  pandas.pydata.org/docs/user_guide/timeseries.html
-    return mapping[freq]
+    items = [mapping[freq]]
+    if multiplier > 1:
+        items.insert(0, str(multiplier))
+    return ''.join(items)
 
 
 def max_pacf(data: pd.DataFrame, group_combinations, target, tss):


Test patch:
diff --git a/tests/integration/advanced/test_timeseries.py b/tests/integration/advanced/test_timeseries.py
--- a/tests/integration/advanced/test_timeseries.py
+++ b/tests/integration/advanced/test_timeseries.py
@@ -350,13 +350,16 @@ def test_5_time_series_arima_mixer(self):
             assert np.allclose(manual_preds, lw_preds, atol=1)
 
     def test_6_time_series_sktime_mixer(self):
-        """ Sanity check with vanilla sktime mixer using a synthetic square wave """
+        """ Sanity check with vanilla sktime mixer using a synthetic square wave sampled at a 15 seconds interval"""
         tsteps = 100
         target = 'Value'
         horizon = 20
+        spacing = 15
         df = pd.DataFrame(columns=['Time', target])
-        df['Time'] = np.linspace(0, 100, tsteps, endpoint=False)
-        df[target] = [i + f for i, f in enumerate(signal.sawtooth(2 * np.pi * 5 * df['Time'].values, width=0.5))]
+        start_ts = datetime(2022, 12, 31, 23, 59, 00)
+        ts_gen = np.linspace(0, 100 * spacing, tsteps, endpoint=False)
+        df['Time'] = [str(start_ts + timedelta(seconds=f)) for f in ts_gen]
+        df[target] = [i + f for i, f in enumerate(signal.sawtooth(2 * np.pi * 5 * ts_gen, width=0.5))]
         df[f'{target}_2x'] = [2 * elt for elt in df[target].values]
         train = df[:int(len(df) * 0.8)]
         test = df[int(len(df) * 0.8):]
@@ -553,7 +556,7 @@ def test_11_output_date_format(self):
                                                                                   }}))
         predictor.learn(train_df)
         preds = predictor.predict(test_df.iloc[[-1]], args={'time_format': 'infer'})
-        self.assertEqual(preds[f'order_{order_by}'].iloc[-1], ['2012-07', '2012-10'])
+        self.assertTrue(preds[f'order_{order_by}'].iloc[-1] in (['2012-06', '2012-09'], ['2012-07', '2012-10']))
 
         preds = predictor.predict(test_df.iloc[[-1]], args={'time_format': '%Y'})
         self.assertEqual(preds[f'order_{order_by}'].iloc[-1], ['2012', '2012'])