Repository: mindsdb/lightwood

Issue:
Trigger TS resample automatically if partitions are malformed
[Reference issue](https://github.com/mindsdb/mindsdb/issues/2600)

Trigger the behavior in #958 automatically if partitions are not valid (i.e. duplicate time entries but different target readings). Important to clearly communicate default policies here because it has implications on what types of `JOIN` are possible once the model is used to predict.


Discussion:

Resolving patch:
diff --git a/lightwood/data/timeseries_transform.py b/lightwood/data/timeseries_transform.py
--- a/lightwood/data/timeseries_transform.py
+++ b/lightwood/data/timeseries_transform.py
@@ -50,6 +50,11 @@ def transform_timeseries(
     # infer frequency with get_delta
     oby_col = tss.order_by
     groups = get_ts_groups(data, tss)
+
+    # initial stable sort and per-partition deduplication
+    data = data.sort_values(by=oby_col, kind='mergesort')
+    data = data.drop_duplicates(subset=[oby_col, *gb_arr], keep='first')
+
     if not ts_analysis:
         _, periods, freqs = get_delta(data, dtype_dict, groups, target, tss)
     else:
diff --git a/lightwood/mixer/sktime.py b/lightwood/mixer/sktime.py
--- a/lightwood/mixer/sktime.py
+++ b/lightwood/mixer/sktime.py
@@ -316,7 +316,7 @@ def _get_best_model(self, trial, train_data, test_data):
         try:
             self._fit(train_data)
             y_true = test_data.data_frame[self.target].values[:self.horizon]
-            y_pred = self(test_data)['prediction'].iloc[0][:len(y_true)]
+            y_pred = pd.DataFrame(self(test_data)['prediction'].iloc[0][:len(y_true)])
             error = self.trial_error_fn(y_true, y_pred)
         except Exception as e:
             log.debug(e)


Test patch:
diff --git a/tests/integration/advanced/test_timeseries.py b/tests/integration/advanced/test_timeseries.py
--- a/tests/integration/advanced/test_timeseries.py
+++ b/tests/integration/advanced/test_timeseries.py
@@ -257,16 +257,16 @@ def test_4_time_series_classification_long_horizon_multiclass(self):
         predictor.learn(train)
         predictor.predict(test)
 
-    def test_5_time_series_sktime_mixer(self):
+    def test_5_time_series_arima_mixer(self):
         """
-        Tests `sktime` mixer individually, as it has a special notion of
-        timestamps that we need to ensure are being used correctly. In
-        particular, given a train-dev-test split, any forecasts coming from a sktime
+        Tests `AutoARIMA` mixer individually, as it has a special notion of
+        timestamps that we need to ensure are being used correctly (along with all other `sktime` mixers). 
+        In particular, given a train-dev-test split, any forecasts coming from any sktime
         mixer should start from the latest observed data in the entire dataset.
         
         This test also compares:
          - correct propagation of offset by K if the special `__mdb_forecast_offset` column is present
-         - results against manual use of sktime to ensure equal results.
+         - results against manual use of AutoARIMA to ensure equal results.
         """  # noqa
 
         # synth square wave
@@ -294,7 +294,7 @@ def test_5_time_series_sktime_mixer(self):
 
         json_ai = json_ai_from_problem(df, problem_definition=pdef)
         json_ai.model['args']['submodels'] = [{
-            "module": "SkTime",
+            "module": "ARIMAMixer",
             "args": {
                 "stop_after": "$problem_definition.seconds_per_mixer",
                 "horizon": "$problem_definition.timeseries_settings.horizon",
@@ -350,7 +350,38 @@ def test_5_time_series_sktime_mixer(self):
             lw_preds = [p[0] for p in ps['prediction']]
             assert np.allclose(manual_preds, lw_preds, atol=1.5)
 
-    def test_6_irregular_series(self):
+    def test_6_time_series_sktime_mixer(self):
+        """ Sanity check with vanilla sktime mixer using a synthetic square wave """
+        tsteps = 100
+        target = 'Value'
+        horizon = 20
+        df = pd.DataFrame(columns=['Time', target])
+        df['Time'] = np.linspace(0, 100, tsteps, endpoint=False)
+        df[target] = [i + f for i, f in enumerate(signal.sawtooth(2 * np.pi * 5 * df['Time'].values, width=0.5))]
+        df[f'{target}_2x'] = [2 * elt for elt in df[target].values]
+        train = df[:int(len(df) * 0.8)]
+        test = df[int(len(df) * 0.8):]
+        pdef = ProblemDefinition.from_dict({'target': target,
+                                            'time_aim': 200,
+                                            'timeseries_settings': {
+                                                'order_by': 'Time',
+                                                'window': 5,
+                                                'horizon': horizon,
+                                                'historical_columns': [f'{target}_2x']
+                                            }})
+        json_ai = json_ai_from_problem(df, problem_definition=pdef)
+        json_ai.model['args']['submodels'] = [{
+            "module": "SkTime",
+            "args": {
+                "stop_after": "$problem_definition.seconds_per_mixer",
+                "horizon": "$problem_definition.timeseries_settings.horizon",
+            }}]
+        predictor = predictor_from_code(code_from_json_ai(json_ai))
+        train_and_check_time_aim(predictor, train)
+        ps = predictor.predict(test)
+        assert r2_score(test[target].values, ps['prediction'].iloc[0]) >= 0.5
+
+    def test_7_irregular_series(self):
         """
         Even though the suggestion is to feed regularly sampled series into predictors, this test can still help us
         catch undesired behavior when the recommendation is not followed.
@@ -390,7 +421,7 @@ def test_6_irregular_series(self):
         test['__mdb_forecast_offset'] = 1
         predictor.predict(test)
 
-    def test_7_time_series_double_grouped_regression(self):
+    def test_8_time_series_double_grouped_regression(self):
         """Test double-grouped numerical predictions, replicates quick start guide in cloud.mindsdb.com """
         data = pd.read_csv('tests/data/house_sales.csv')
         gby = ['bedrooms', 'type']
@@ -422,3 +453,22 @@ def test_7_time_series_double_grouped_regression(self):
             row[f'order_{order_by}'] = [row[f'order_{order_by}']] if horizon == 1 else row[f'order_{order_by}']
             for timestamp in row[f'order_{order_by}']:
                 assert timestamp > pd.to_datetime(test[order_by]).max().timestamp()
+
+    def test_9_ts_dedupe(self):
+        """ Test time series de-duplication procedures """
+        data = pd.read_csv('tests/data/arrivals.csv')
+        data = data[data['Country'].isin(['US', 'Japan'])]
+        target_len = len(data)
+        data = data.append(data[data['Country'] == 'Japan']).reset_index(drop=True)  # force duplication of one series
+        jai = json_ai_from_problem(data, ProblemDefinition.from_dict({'target': 'Traffic',
+                                                                      'time_aim': 30,
+                                                                      'timeseries_settings': {
+                                                                          'group_by': ['Country'],
+                                                                          'horizon': 8,
+                                                                          'order_by': 'T',
+                                                                          'window': 4
+                                                                      }}))
+        code = code_from_json_ai(jai)
+        pred = predictor_from_code(code)
+        transformed = pred.preprocess(data)
+        assert len(transformed) == target_len