Repository: mindsdb/mindsdb

Issue:
Add mistakes to tests
We should add "mistakes" and "crashes" to a bunch of the tests, especially those specified in #898.

## I - Crash during tests

1. Start the tested action (e.g. training a model, uploading a dataset)
2. Before it finishes, intrerupt it using a separate process that kills mindsdb
3. Make sure mindsdb can start from the failed state and try the actions again

## II - Mistakes during tests

1. Add a test step that has a mistake (e.g. wrong password for integration, wrong columns for prediction)
2. Make sure a failure is obvious to the user once this step is done
3. Make sure mindsdb still works and can be stopped and started back up again without issues
4. Redo step 1) correctly and make sure everything is fine.


Discussion:

Resolving patch:
diff --git a/mindsdb/api/http/namespaces/predictor.py b/mindsdb/api/http/namespaces/predictor.py
--- a/mindsdb/api/http/namespaces/predictor.py
+++ b/mindsdb/api/http/namespaces/predictor.py
@@ -150,12 +150,15 @@ def put(self, name):
                 retrain = True
             else:
                 retrain = False
-        except:
+        except Exception:
             retrain = None
 
         ds_name = data.get('data_source_name') if data.get('data_source_name') is not None else data.get('from_data')
         from_data = ca.default_store.get_datasource_obj(ds_name, raw=True)
 
+        if from_data is None:
+            return {'message': f'Can not find datasource: {ds_name}'}, 400
+
         if retrain is True:
             original_name = name
             name = name + '_retrained'
diff --git a/mindsdb/utilities/functions.py b/mindsdb/utilities/functions.py
--- a/mindsdb/utilities/functions.py
+++ b/mindsdb/utilities/functions.py
@@ -42,7 +42,7 @@ def get_all_models_meta_data(mindsdb_native, custom_models):
             'name': x['name'],
             'predict': x['predict'],
             'data_analysis': mindsdb_native.get_model_data(x['name'])['data_analysis_v2']
-        } for x in mindsdb_native.get_models()
+        } for x in mindsdb_native.get_models() if x['status'] == 'complete'
     ]
 
     model_data_arr.extend(custom_models.get_models())


Test patch:
diff --git a/tests/integration_tests/flows/http_test_helpers.py b/tests/integration_tests/flows/http_test_helpers.py
--- a/tests/integration_tests/flows/http_test_helpers.py
+++ b/tests/integration_tests/flows/http_test_helpers.py
@@ -59,7 +59,7 @@ def check_ds_analyzable(ds_name):
 def wait_predictor_learn(predictor_name):
     start_time = time.time()
     learn_done = False
-    while learn_done is False and (time.time() - start_time) < 60:
+    while learn_done is False and (time.time() - start_time) < 120:
         learn_done = get_predictor_data(predictor_name)['status'] == 'complete'
         time.sleep(1)
     assert learn_done
diff --git a/tests/integration_tests/flows/test_mistakes.py b/tests/integration_tests/flows/test_mistakes.py
new file mode 100644
--- /dev/null
+++ b/tests/integration_tests/flows/test_mistakes.py
@@ -0,0 +1,221 @@
+import unittest
+import requests
+import asyncio
+import time
+
+from mindsdb.utilities.config import Config
+
+from common import (
+    MINDSDB_DATABASE,
+    HTTP_API_ROOT,
+    TEST_CONFIG,
+    run_environment,
+    stop_mindsdb
+)
+
+from http_test_helpers import (
+    wait_predictor_learn,
+    check_predictor_not_exists,
+    check_ds_not_exists,
+    check_ds_exists
+)
+
+# +++ define test data
+TEST_DATASET = 'us_health_insurance'
+
+TO_PREDICT = {
+    'smoker': str
+}
+CONDITION = {
+    'age': 20,
+    'sex': 'female'
+}
+# ---
+
+TEST_DATA_TABLE = TEST_DATASET
+TEST_PREDICTOR_NAME = f'{TEST_DATASET}_predictor'
+EXTERNAL_DS_NAME = f'{TEST_DATASET}_external'
+
+TEST_INTEGRATION = 'test_integration'
+TEST_DS = 'test_ds'
+TEST_PREDICTOR = 'test_predictor'
+
+config = Config(TEST_CONFIG)
+
+
+class UserFlowTest_1(unittest.TestCase):
+    def test_1_wrong_integration(self):
+        '''
+        start mindsdb with enabled integration with wrong password
+        try create ds
+        change password to correct
+        '''
+        original_db_password = config['integrations']['default_mariadb']['password']
+        self.mdb, datastore = run_environment(
+            config,
+            apis=['mysql', 'http'],
+            override_integration_config={
+                'default_mariadb': {
+                    'enabled': True,
+                    'password': 'broken'
+                }
+            },
+            mindsdb_database=MINDSDB_DATABASE
+        )
+
+        check_ds_not_exists(TEST_DS)
+
+        # TODO creating DS from unexists integration raise not critical error in code.
+        # need fix it and return human-readable error
+        # related issue: https://github.com/mindsdb/mindsdb/issues/945
+        # data = {
+        #     "integration_id": 'unexists_integration',
+        #     "name": TEST_DS,
+        #     "query": f"select * from test_data.{TEST_DATASET} limit 50;"
+        # }
+        # res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
+        # assert res ?
+
+        # check create DS with wrong integration password
+        data = {
+            "integration_id": 'default_mariadb',
+            "name": TEST_DS,
+            "query": f"select * from test_data.{TEST_DATASET} limit 100;"
+        }
+        res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
+        assert 'Access denied for user' in res.json()['message']
+
+        check_ds_not_exists(TEST_DS)
+
+        # restore password
+        res = requests.post(
+            f'{HTTP_API_ROOT}/config/integrations/default_mariadb',
+            json={'params': {'password': original_db_password}}
+        )
+        assert res.status_code == 200
+        config['integrations']['default_mariadb']['password'] = original_db_password
+
+    def test_2_broke_analisys(self):
+        '''
+        stop mindsdb while analyse dataset
+        '''
+        data = {
+            "integration_id": 'default_mariadb',
+            "name": TEST_DS,
+            "query": f"select * from test_data.{TEST_DATASET} limit 100;"
+        }
+        res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
+        assert res.status_code == 200
+
+        res = requests.get(f'{HTTP_API_ROOT}/datasources/{TEST_DS}/analyze')
+        assert res.status_code == 200
+
+        stop_mindsdb()
+
+        self.mdb, datastore = run_environment(
+            config,
+            apis=['mysql', 'http'],
+            override_integration_config={
+                'default_mariadb': {
+                    'enabled': True
+                }
+            },
+            mindsdb_database=MINDSDB_DATABASE,
+            clear_storage=False
+        )
+
+        check_ds_exists(TEST_DS)
+
+    def test_3_wrong_predictor(self):
+        '''
+        try create predictor with wrong parameters,
+        close mindsdb while model training
+        check mindsdb can start again
+        '''
+        check_predictor_not_exists(TEST_PREDICTOR)
+
+        data = {
+            'to_predict': list(TO_PREDICT.keys()),
+            'data_source_name': 'wrong ds'
+        }
+        res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}', json=data)
+        assert 'Can not find datasource' in res.json()['message']
+
+        check_predictor_not_exists(TEST_PREDICTOR)
+
+        data = {
+            'to_predict': list(TO_PREDICT.keys()),
+            'data_source_name': TEST_DS
+        }
+        res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}', json=data)
+        assert res.status_code == 200
+
+        stop_mindsdb()
+
+        self.mdb, datastore = run_environment(
+            config,
+            apis=['mysql', 'http'],
+            override_integration_config={
+                'default_mariadb': {
+                    'enabled': True
+                }
+            },
+            mindsdb_database=MINDSDB_DATABASE,
+            clear_storage=False
+        )
+
+        # TODO add after this issue will be closed: https://github.com/mindsdb/mindsdb/issues/948
+        # check_predictor_not_exists(TEST_PREDICTOR)
+
+        data = {
+            'to_predict': list(TO_PREDICT.keys()),
+            'data_source_name': TEST_DS
+        }
+        res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2', json=data)
+        assert res.status_code == 200
+
+        wait_predictor_learn(f'{TEST_PREDICTOR}_2')
+
+    def test_4_wrong_prediction(self):
+        '''
+        close mindsdb while make prediction, then try run it again
+        '''
+        ioloop = asyncio.get_event_loop()
+        if ioloop.is_closed():
+            ioloop = asyncio.new_event_loop()
+        ioloop.run_in_executor(
+            None,
+            lambda: requests.post(
+                f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2/predict',
+                json={'when': CONDITION}
+            )
+        )
+        time.sleep(0.5)
+        stop_mindsdb()
+        ioloop.close()
+
+        self.mdb, datastore = run_environment(
+            config,
+            apis=['mysql', 'http'],
+            override_integration_config={
+                'default_mariadb': {
+                    'enabled': True
+                }
+            },
+            mindsdb_database=MINDSDB_DATABASE,
+            clear_storage=False
+        )
+
+        res = requests.post(
+            f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2/predict',
+            json={'when': CONDITION}
+        )
+        assert res.status_code == 200
+
+
+if __name__ == "__main__":
+    try:
+        unittest.main(failfast=True)
+        print('Tests passed!')
+    except Exception as e:
+        print(f'Tests Failed!\n{e}')