Repository: mindsdb/mindsdb

Issue:
Feedback predict column into timeseries
We should be feeding back the previous values of the `predict` column as a sequential column when making timeseries predictions.

Consider doing this by using the ludwig model (label1, label2, label3... etc as separate columns,  rather than 1 large array), at least when using the ludwig backend.

This could also cause issues in certain predict cases, depending on how we train our model. So we should train the model to handle both the case when predictions are available and the case when they aren't or provide an argument for it in the `Preidctor` or in the `learn` interface.


Discussion:

Resolving patch:
diff --git a/mindsdb/libs/backends/ludwig.py b/mindsdb/libs/backends/ludwig.py
--- a/mindsdb/libs/backends/ludwig.py
+++ b/mindsdb/libs/backends/ludwig.py
@@ -12,26 +12,50 @@ class LudwigBackend():
     def __init__(self, transaction):
         self.transaction = transaction
 
-    def _translate_df_to_timeseries_format(self, df, model_definition, timeseries_cols):
-        input_features = model_definition['input_features']
+    def _translate_df_to_timeseries_format(self, df, model_definition, timeseries_cols, mode='predict'):
+        timeseries_col_name = timeseries_cols[0]
+
+        previous_predict_col_names = []
+        predict_col_names = []
+        for feature_def in model_definition['output_features']:
+            if mode == 'train':
+                predict_col_names.append(feature_def['name'])
+            else:
+                predict_col_names = []
+            previous_predict_col_name = 'previous_' + feature_def['name']
+
+            previous_predict_col_already_in = False
+
+            for definition in model_definition['input_features']:
+                if definition['name'] == previous_predict_col_name:
+                     previous_predict_col_already_in = True
+
+            if not previous_predict_col_already_in:
+                model_definition['input_features'].append({
+                    'name': previous_predict_col_name
+                    ,'type': 'sequence'
+                })
 
         other_col_names = []
-        timeseries_col_name = timeseries_cols[0]
-        for feature_def in input_features:
-            if feature_def['name'] not in self.transaction.persistent_model_metadata.model_group_by:
+        for feature_def in model_definition['input_features']:
+            if feature_def['name'] not in self.transaction.persistent_model_metadata.model_group_by and feature_def['name'] not in previous_predict_col_names:
                 feature_def['type'] = 'sequence'
                 if feature_def['name'] not in timeseries_cols:
                     other_col_names.append(feature_def['name'])
 
+
+            previous_predict_col_names.append(previous_predict_col_name)
+
         new_cols = {}
-        for col in [*other_col_names,timeseries_col_name]:
+        for col in [*other_col_names,*previous_predict_col_names,timeseries_col_name,*predict_col_names,*self.transaction.persistent_model_metadata.model_group_by]:
             new_cols[col] = []
 
         nr_ele = len(df[timeseries_col_name])
 
         if self.transaction.persistent_model_metadata.window_size_seconds is not None:
             window_size_seconds = self.transaction.persistent_model_metadata.window_size_seconds
-            for i in range(nr_ele):
+            i = 0
+            while i < nr_ele:
                 current_window = 0
                 new_row = {}
 
@@ -39,50 +63,94 @@ def _translate_df_to_timeseries_format(self, df, model_definition, timeseries_co
 
                 for col in other_col_names:
                     new_row[col] = [df[col][i]]
+                for col in previous_predict_col_names:
+                    new_row[col] = []
+                for col in predict_col_names:
+                    new_row[col] = df[col][i]
+                for col in self.transaction.persistent_model_metadata.model_group_by:
+                    new_row[col] = df[col][i]
 
                 inverted_index_range = list(range(i))
                 inverted_index_range.reverse()
+                ii = 0
                 for ii in inverted_index_range:
                     if window_size_seconds < current_window + (timeseries_row[-1] - df[timeseries_col_name][ii]):
                         break
                     current_window += (timeseries_row[-1] - df[timeseries_col_name][ii])
                     timeseries_row.append(df[timeseries_col_name][ii])
+
                     for col in other_col_names:
                         new_row[col].append(df[col][ii])
+                    for col in previous_predict_col_names:
+                        try:
+                            new_row[col].append(df[col.replace('previous_', '')][ii])
+                        except:
+                            try:
+                                new_row[col].append(df[col][ii])
+                            except:
+                                self.transaction.log.warning('Missing previous predicted values for output column: {}, these should be included in your input under the name: {}'.format(col.replace('previous_', ''), col))
+
+                if mode == 'train':
+                    i = max(i + 1, (i + round((i - ii)/2)))
+                else:
+                    i = i + 1
 
                 new_row[timeseries_col_name] = timeseries_row
 
                 for col in new_row:
-                    new_row[col].reverse()
+                    if col not in predict_col_names and col not in self.transaction.persistent_model_metadata.model_group_by:
+                        new_row[col].reverse()
                     new_cols[col].append(new_row[col])
         else:
             window_size_samples = self.transaction.persistent_model_metadata.window_size_samples
-            for i in range(nr_ele):
+            i = 0
+            while i < nr_ele:
                 new_row = {}
 
                 timeseries_row = [df[timeseries_col_name][i]]
 
                 for col in other_col_names:
                     new_row[col] = [df[col][i]]
+                for col in previous_predict_col_names:
+                    new_row[col] = []
+                for col in predict_col_names:
+                    new_row[col] = df[col][i]
+                for col in self.transaction.persistent_model_metadata.model_group_by:
+                    new_row[col] = df[col][i]
 
                 inverted_index_range = list(range(i))
                 inverted_index_range.reverse()
+                ii = 0
                 for ii in inverted_index_range:
                     if (i - ii) > window_size_samples:
                         break
                     timeseries_row.append(df[timeseries_col_name][ii])
+
                     for col in other_col_names:
                         new_row[col].append(df[col][ii])
+                    for col in previous_predict_col_names:
+                        try:
+                            new_row[col].append(df[col.replace('previous_', '')][ii])
+                        except:
+                            try:
+                                new_row[col].append(df[col][ii])
+                            except:
+                                self.transaction.log.warning('Missing previous predicted values for output column: {}, these should be included in your input under the name: {}'.format(col.replace('previous_', ''), col))
+
+                if mode == 'train':
+                    i = max(i + 1, (i + round((i - ii)/2)))
+                else:
+                    i = i + 1
 
                 new_row[timeseries_col_name] = timeseries_row
 
                 for col in new_row:
-                    new_row[col].reverse()
+                    if col not in predict_col_names and col not in self.transaction.persistent_model_metadata.model_group_by:
+                        new_row[col].reverse()
                     new_cols[col].append(new_row[col])
 
-
-        for col in new_cols:
-            df[col] = new_cols[col]
+        new_df = pd.DataFrame(data=new_cols)
+        df = new_df
         return df, model_definition
 
     def _create_ludwig_dataframe(self, mode):
@@ -94,10 +162,10 @@ def _create_ludwig_dataframe(self, mode):
             columns = [[col, col_ind] for col_ind, col in enumerate(self.transaction.persistent_model_metadata.columns) if col not in self.transaction.persistent_model_metadata.predict_columns]
         elif mode == 'validate':
             indexes = self.transaction.input_data.validation_indexes[KEY_NO_GROUP_BY]
-            columns = [[col, col_ind] for col_ind, col in enumerate(self.transaction.persistent_model_metadata.columns) if col not in self.transaction.persistent_model_metadata.predict_columns]
+            columns = [[col, col_ind] for col_ind, col in enumerate(self.transaction.persistent_model_metadata.columns)]  #if col not in self.transaction.persistent_model_metadata.predict_columns]
         elif mode == 'test':
             indexes = self.transaction.input_data.test_indexes[KEY_NO_GROUP_BY]
-            columns = [[col, col_ind] for col_ind, col in enumerate(self.transaction.persistent_model_metadata.columns) if col not in self.transaction.persistent_model_metadata.predict_columns]
+            columns = [[col, col_ind] for col_ind, col in enumerate(self.transaction.persistent_model_metadata.columns)] #if col not in self.transaction.persistent_model_metadata.predict_columns]
         else:
             raise Exception(f'Unknown mode specified: "{mode}"')
         model_definition = {'input_features': [], 'output_features': []}
@@ -283,7 +351,7 @@ def train(self):
             timeseries_cols = list(map(lambda x: x[0], self.transaction.persistent_model_metadata.model_order_by))
 
         if len(timeseries_cols) > 0:
-            training_dataframe, model_definition =  self._translate_df_to_timeseries_format(training_dataframe, model_definition, timeseries_cols)
+            training_dataframe, model_definition =  self._translate_df_to_timeseries_format(training_dataframe, model_definition, timeseries_cols, 'train')
 
         model = LudwigModel(model_definition)
 
@@ -301,6 +369,7 @@ def train(self):
 
     def predict(self, mode='predict', ignore_columns=[]):
         predict_dataframe, model_definition = self._create_ludwig_dataframe(mode)
+        model_definition = self.transaction.persistent_model_metadata.ludwig_data['model_definition']
 
         model = LudwigModel.load(self.transaction.persistent_model_metadata.ludwig_data['ludwig_save_path'])
 
@@ -320,6 +389,7 @@ def predict(self, mode='predict', ignore_columns=[]):
                     predict_dataframe[ignore_col + date_appendage] = [None] * len(predict_dataframe[ignore_col + date_appendage])
 
         predictions = model.predict(data_df=predict_dataframe)
+
         for col_name in predictions:
             col_name_normalized = col_name.replace('_predictions', '')
             predictions = predictions.rename(columns = {col_name: col_name_normalized})
diff --git a/mindsdb/libs/helpers/general_helpers.py b/mindsdb/libs/helpers/general_helpers.py
--- a/mindsdb/libs/helpers/general_helpers.py
+++ b/mindsdb/libs/helpers/general_helpers.py
@@ -100,7 +100,7 @@ def check_for_updates():
     if file_path.is_file():
         token = open(mdb_file).read()
     else:
-        token = '{system}|{version}|{uid}'.format(system=platform.system(), version=__version, uid=uuid_str)
+        token = '{system}|{version}|{uid}'.format(system=platform.system(), version=__version__, uid=uuid_str)
         try:
             open(mdb_file,'w').write(token)
         except:


Test patch:
diff --git a/integration_testing/run_tests.py b/integration_testing/run_tests.py
--- a/integration_testing/run_tests.py
+++ b/integration_testing/run_tests.py
@@ -14,8 +14,8 @@
 import mindsdb
 from mindsdb import CONST
 
-types_that_fail = ['str','ascii']
-types_that_work = ['int','float','date','datetime','timestamp']
+types_that_fail = ['str']
+types_that_work = ['int','float','date','datetime','timestamp','ascii']
 
 logger = None
 
@@ -53,7 +53,8 @@ def test_timeseries():
     logger.debug(f'Creating timeseries test datasets and saving them to {train_file_name} and {test_file_name}, total dataset size will be {data_len} rows')
 
     try:
-        features = generate_value_cols(['datetime','int','float','ascii','ascii'],data_len, separator, ts_hours * 3600)
+        # add ,'ascii' in the features list to re-implement the group by
+        features = generate_value_cols(['datetime','int','float', 'ascii'],data_len, separator, ts_hours * 3600)
         features[3] = list(map(lambda x: str(x[0]) if len(x) > 0 else 'Nrmm', features[3]))
         labels = [generate_labels_1(features, separator)]
 
@@ -92,6 +93,7 @@ def test_timeseries():
             # timeseries specific argsw
             ,order_by=feature_headers[0]
             ,window_size_seconds=ts_hours* 3600 * 1.5
+            #,window_size=6
             ,group_by = feature_headers[3]
         )
         logger.info(f'--------------- Learning ran succesfully ---------------')
@@ -114,6 +116,7 @@ def test_timeseries():
         for row in results:
             expect_columns = [label_headers[0] ,label_headers[0] + '_confidence']
             for col in expect_columns:
+                print(col, row[col])
                 if col not in row:
                     logger.error(f'Prediction failed to return expected column: {col}')
                     logger.debug('Got row: {}'.format(row))
@@ -367,7 +370,7 @@ def test_multilabel_prediction():
 
 
 setup_testing_logger()
-#test_one_label_prediction_wo_strings()
-#test_timeseries()
-#test_multilabel_prediction()
+test_one_label_prediction_wo_strings()
+test_timeseries()
+test_multilabel_prediction()
 test_one_label_prediction()
diff --git a/integration_testing/run_travis_tests.py b/integration_testing/run_travis_tests.py
new file mode 100644
--- /dev/null
+++ b/integration_testing/run_travis_tests.py
@@ -0,0 +1,117 @@
+from data_generators import *
+import traceback
+import sys
+import os
+import itertools
+import logging
+from colorlog import ColoredFormatter
+import time
+
+import mindsdb
+from mindsdb import CONST
+
+types_that_work = ['int','float','date','datetime','timestamp','ascii']
+
+logger = None
+
+def setup_testing_logger():
+    global logger
+    formatter = ColoredFormatter(
+        "%(log_color)s%(message)s",
+        datefmt=None,
+        reset=True,
+        log_colors={
+            'DEBUG':    'black,bg_white',
+            'INFO':     'blue,bg_white',
+            'WARNING':  'orange,bg_white',
+            'ERROR':    'red,bg_white',
+            'CRITICAL': 'red,bg_white',
+        }
+    )
+
+    logger = logging.getLogger('mindsdb_integration_testing')
+    logger.handlers = []
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+
+def run_tests():
+    logger.info('Starting one-label test')
+    separator = ','
+    train_file_name = 'train_data.csv'
+    test_file_name = 'test_data.csv'
+    data_len = 8000
+
+    # Create the full dataset
+    logger.debug(f'Creating one-labe test datasets and saving them to {train_file_name} and {test_file_name}, total dataset size will be {data_len} rows')
+
+    try:
+        features = generate_value_cols(types_that_work,data_len, separator)
+        labels = [generate_labels_2(features, separator)]
+
+        feature_headers = list(map(lambda col: col[0], features))
+        label_headers = list(map(lambda col: col[0], labels))
+
+        # Create the training dataset and save it to a file
+        columns_train = list(map(lambda col: col[1:int(len(col)*3/4)], features))
+        columns_train.extend(list(map(lambda col: col[1:int(len(col)*3/4)], labels)))
+        columns_to_file(columns_train, train_file_name, separator, headers=[*feature_headers,*label_headers])
+
+        # Create the testing dataset and save it to a file
+        columns_test = list(map(lambda col: col[int(len(col)*3/4):], features))
+        columns_to_file(columns_test, test_file_name, separator, headers=feature_headers)
+        logger.debug(f'Datasets generate and saved to files successfully')
+    except:
+        print(traceback.format_exc())
+        logger.error(f'Failed to generate datasets !')
+        exit(1)
+
+    # Train
+    mdb = None
+    try:
+        mdb = mindsdb.Predictor(name='test_one_label_prediction')
+        logger.debug(f'Succesfully create mindsdb Predictor')
+    except:
+        logger.error(f'Failed to create mindsdb Predictor')
+        exit(1)
+
+
+    try:
+        mdb.learn(from_data=train_file_name, to_predict=label_headers)
+        logger.info(f'--------------- Learning ran succesfully ---------------')
+    except:
+        print(traceback.format_exc())
+        logger.error(f'Failed during the training !')
+        exit(1)
+
+    # Predict
+    try:
+        mdb = mindsdb.Predictor(name='test_one_label_prediction')
+        logger.debug(f'Succesfully create mindsdb Predictor')
+    except:
+        print(traceback.format_exc())
+        logger.error(f'Failed to create mindsdb Predictor')
+        exit(1)
+
+    try:
+        results = mdb.predict(when_data=test_file_name)
+        for row in results:
+            expect_columns = [label_headers[0] ,label_headers[0] + '_confidence']
+            for col in expect_columns:
+                if col not in row:
+                    logger.error(f'Prediction failed to return expected column: {col}')
+                    logger.debug('Got row: {}'.format(row))
+                    exit(1)
+
+        logger.info(f'--------------- Predicting ran succesfully ---------------')
+    except:
+        print(traceback.format_exc())
+        logger.error(f'Failed whilst predicting')
+        exit(1)
+
+    logger.info('Travis CLI Tests ran succesfully !')
+
+
+setup_testing_logger()
+run_tests()