Repository: mindsdb/mindsdb

Issue:
Complete clickhouse & mariadb integration testing and make sure it's running as part of the CI
The integration test for clickhouse & mariadb should include:

1. (setup the env) Use docker compose to boot up the databases + Upload some data + start mindsdb 
2. Test that all mindsdb tables are created (commands, predictors, the mindsdb db)
3. Train a predictor from an internal datasource (using some kwargs to validate they work)
4. Train a predictor from an external datasource (say, a file)
5. With both predictors from steps 3 and 4:
* Predict from a `WHEN` statement
* Predict from a query internal to the databases
* Predict from an external datasource

For point 5. please specify some kwargs to the predictions to, again, validate that those also work.


Discussion:

Resolving patch:
diff --git a/mindsdb/api/mysql/mysql_proxy/classes/sql_query.py b/mindsdb/api/mysql/mysql_proxy/classes/sql_query.py
--- a/mindsdb/api/mysql/mysql_proxy/classes/sql_query.py
+++ b/mindsdb/api/mysql/mysql_proxy/classes/sql_query.py
@@ -59,12 +59,10 @@ def parse_insert(sql):
 
         return dict(zip(columns, values))
 
-    def __init__(self, sql, session=None):
+    def __init__(self, sql, integration=None, database=None):
         # parse
-        self.session = session
-        self.integration = None
-        if session is not None:
-            self.integration = session.integration
+        self.integration = integration
+        self.database = database
 
         # 'offset x, y' - specific just for mysql, parser dont understand it
         sql = re.sub(r'\n?limit([\n\d\s]*),([\n\d\s]*)', ' limit \g<1> offset \g<1> ', sql)
@@ -121,9 +119,7 @@ def _format_from_statement(self, s):
                     {'value': 'xxx.zzz_mariadb', 'name': 'a'}
                     -> {'value': 'xxx.zzz', 'name': 'a', source: 'mariadb'}
         """
-        database = None
-        if self.session is not None:
-            database = self.session.database
+        database = self.database
         if isinstance(s, str):
             if '.' in s:
                 s = {
@@ -423,7 +419,7 @@ def _fetchData(self):
                 result = []
                 if 'select ' not in external_datasource.lower():
                     external_datasource = f'select * from {external_datasource}'
-                query = SQLQuery(external_datasource, default_dn='datasource')
+                query = SQLQuery(external_datasource, database='datasource', integration=self.integration)
                 result = query.fetch(self.datahub, view='dict')
                 if result['success'] is False:
                     raise Exception(result['msg'])
diff --git a/mindsdb/api/mysql/mysql_proxy/controllers/session_controller.py b/mindsdb/api/mysql/mysql_proxy/controllers/session_controller.py
--- a/mindsdb/api/mysql/mysql_proxy/controllers/session_controller.py
+++ b/mindsdb/api/mysql/mysql_proxy/controllers/session_controller.py
@@ -27,6 +27,9 @@ def __init__(self) -> object:
         self.auth = False
         self.logging = log
 
+        self.integration = None
+        self.database = None
+
         self.current_transaction = None
 
     def newTransaction(self, sql_query):
diff --git a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
--- a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
+++ b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
@@ -362,7 +362,7 @@ def delete_predictor_answer(self, sql):
 
         fake_sql = sql.strip(' ')
         fake_sql = 'select name ' + fake_sql[len('delete '):]
-        query = SQLQuery(fake_sql, session=self.session)
+        query = SQLQuery(fake_sql, integration=self.session.integration, database=self.session.database)
 
         result = query.fetch(datahub)
 
@@ -499,7 +499,7 @@ def queryAnswer(self, sql):
             if 'database()' in sql_lower:
                 self.answerSelectDatabase()
                 return
-            query = SQLQuery(sql, session=self.session)
+            query = SQLQuery(sql, integration=self.session.integration, database=self.session.database)
             return self.selectAnswer(query)
         elif keyword == 'rollback':
             self.packet(OkPacket).send()


Test patch:
diff --git a/tests/docker/mariadb/connect.cnf b/tests/docker/mariadb/connect.cnf
--- a/tests/docker/mariadb/connect.cnf
+++ b/tests/docker/mariadb/connect.cnf
@@ -2,3 +2,6 @@
 plugin-load-add=ha_connect.so
 connect_jvm_path="/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/amd64/server"
 connect_class_path="/usr/lib/mysql/plugin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/ext/dremio-jdbc-driver.jar"
+wait_timeout=180
+connect_timeout=180
+interactive_timeout=180
\ No newline at end of file
diff --git a/tests/integration_tests/flows/common.py b/tests/integration_tests/flows/common.py
--- a/tests/integration_tests/flows/common.py
+++ b/tests/integration_tests/flows/common.py
@@ -1,22 +1,23 @@
 import psutil
 import shutil
 import time
-import pathlib
+from pathlib import Path
 import json
 import docker
+import requests
 import subprocess
 
 from mindsdb.interfaces.database.database import DatabaseWrapper
 
 TEST_CONFIG = 'tests/integration_tests/flows/config/config.json'
 
-TESTS_ROOT = pathlib.Path(__file__).parent.absolute().joinpath('../../').resolve()
+TESTS_ROOT = Path(__file__).parent.absolute().joinpath('../../').resolve()
 
 START_TIMEOUT = 15
 
 OUTPUT = None  # [None|subprocess.DEVNULL]
 
-TEMP_DIR = pathlib.Path(__file__).parent.absolute().joinpath('../../temp/').resolve()
+TEMP_DIR = Path(__file__).parent.absolute().joinpath('../../temp/').resolve()
 TEMP_DIR.mkdir(parents=True, exist_ok=True)
 
 
@@ -87,3 +88,22 @@ def is_container_run(name):
     containers = docker_client.containers.list()
     containers = [x.name for x in containers if x.status == 'running']
     return name in containers
+
+
+def get_test_csv(name, url, lines_count=None, rewrite=False):
+    test_csv_path = TESTS_ROOT.joinpath('temp/', name).resolve()
+    if not test_csv_path.is_file() or rewrite:
+        r = requests.get(url)
+        with open(test_csv_path, 'wb') as f:
+            f.write(r.content)
+        if lines_count is not None:
+            fp = str(test_csv_path)
+            p = subprocess.Popen(
+                f"mv {fp} {fp}_2; sed -n '1,{lines_count}p' {fp}_2 >> {fp}; rm {fp}_2",
+                cwd=TESTS_ROOT.resolve(),
+                stdout=OUTPUT,
+                stderr=OUTPUT,
+                shell=True
+            )
+            p.wait()
+    return str(test_csv_path)
diff --git a/tests/integration_tests/flows/test_clickhouse.py b/tests/integration_tests/flows/test_clickhouse.py
--- a/tests/integration_tests/flows/test_clickhouse.py
+++ b/tests/integration_tests/flows/test_clickhouse.py
@@ -1,15 +1,16 @@
 import unittest
 import requests
-import os
 import csv
 import inspect
 import subprocess
 import atexit
 
 from mindsdb.interfaces.native.mindsdb import MindsdbNative
+from mindsdb.interfaces.datastore.datastore import DataStore
 from mindsdb.utilities.config import Config
 
 from common import (
+    get_test_csv,
     wait_api_ready,
     prepare_config,
     wait_db,
@@ -26,6 +27,8 @@
 TEST_DATA_TABLE = 'home_rentals'
 TEST_PREDICTOR_NAME = 'test_predictor'
 
+EXTERNAL_DS_NAME = 'test_external'
+
 config = Config(TEST_CONFIG)
 
 
@@ -105,6 +108,7 @@ def setUpClass(cls):
             raise Exception()
 
         cls.mdb = MindsdbNative(config)
+        datastore = DataStore(config)
 
         models = cls.mdb.get_models()
         models = [x['name'] for x in models]
@@ -114,6 +118,9 @@ def setUpClass(cls):
         query('create database if not exists test')
         test_tables = query('show tables from test')
         test_tables = [x['name'] for x in test_tables]
+
+        test_csv_path = get_test_csv(TEST_CSV['name'], TEST_CSV['url'])
+
         if TEST_DATA_TABLE not in test_tables:
             print('creating test data table...')
             query(f'''
@@ -132,12 +139,6 @@ def setUpClass(cls):
                 PARTITION BY location
             ''')
 
-            test_csv_path = str(TESTS_ROOT.joinpath('temp/', TEST_CSV['name']).resolve())
-            if os.path.isfile(test_csv_path) is False:
-                r = requests.get(TEST_CSV['url'])
-                with open(test_csv_path, 'wb') as f:
-                    f.write(r.content)
-
             with open(test_csv_path) as f:
                 csvf = csv.reader(f)
                 i = 0
@@ -165,6 +166,12 @@ def setUpClass(cls):
                     i += 1
             print('done')
 
+        ds = datastore.get_datasource(EXTERNAL_DS_NAME)
+        if ds is not None:
+            datastore.delete_datasource(EXTERNAL_DS_NAME)
+        short_csv_file_path = get_test_csv(f'{EXTERNAL_DS_NAME}.csv', TEST_CSV['url'], lines_count=300, rewrite=True)
+        datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv', short_csv_file_path)
+
     def test_1_initial_state(self):
         print(f'\nExecuting {inspect.stack()[0].function}')
         print('Check all testing objects not exists')
@@ -211,8 +218,8 @@ def test_2_insert_predictor(self):
         mindsdb_tables = [x['name'] for x in mindsdb_tables]
         self.assertTrue(TEST_PREDICTOR_NAME in mindsdb_tables)
 
-    def test_3_insert_predictor_with_existing_ds(self):
-        name = f'{TEST_PREDICTOR_NAME}_2'
+    def test_3_externael_ds(self):
+        name = f'{TEST_PREDICTOR_NAME}_external'
         models = self.mdb.get_models()
         models = [x['name'] for x in models]
         if name in models:
@@ -223,7 +230,7 @@ def test_3_insert_predictor_with_existing_ds(self):
             (
                 '{name}',
                 'rental_price, location',
-                '{TEST_PREDICTOR_NAME}',
+                '{EXTERNAL_DS_NAME}',
                 '{{"join_learn_process": true, "stop_training_in_x_seconds": 3}}'
             );
         """)
@@ -238,6 +245,19 @@ def test_3_insert_predictor_with_existing_ds(self):
         mindsdb_tables = [x['name'] for x in mindsdb_tables]
         self.assertTrue(name in mindsdb_tables)
 
+        res = query(f"""
+            select
+                rental_price, location, sqft, number_of_rooms,
+                rental_price_confidence, rental_price_min, rental_price_max, rental_price_explain
+            from
+                mindsdb.{name} where external_datasource='{EXTERNAL_DS_NAME}'
+        """)
+
+        print('check result')
+        self.assertTrue(len(res) > 0)
+        self.assertTrue(res[0]['rental_price'] is not None and res[0]['rental_price'] != 'None')
+        self.assertTrue(res[0]['location'] is not None and res[0]['location'] != 'None')
+
     def test_4_query_predictor(self):
         print(f'\nExecuting {inspect.stack()[0].function}')
         res = query(f"""
diff --git a/tests/integration_tests/flows/test_mariadb.py b/tests/integration_tests/flows/test_mariadb.py
--- a/tests/integration_tests/flows/test_mariadb.py
+++ b/tests/integration_tests/flows/test_mariadb.py
@@ -1,6 +1,4 @@
 import unittest
-import requests
-import os
 import csv
 import inspect
 import subprocess
@@ -9,9 +7,11 @@
 import mysql.connector
 
 from mindsdb.interfaces.native.mindsdb import MindsdbNative
+from mindsdb.interfaces.datastore.datastore import DataStore
 from mindsdb.utilities.config import Config
 
 from common import (
+    get_test_csv,
     wait_api_ready,
     prepare_config,
     wait_db,
@@ -28,6 +28,8 @@
 TEST_DATA_TABLE = 'home_rentals'
 TEST_PREDICTOR_NAME = 'test_predictor'
 
+EXTERNAL_DS_NAME = 'test_external'
+
 config = Config(TEST_CONFIG)
 
 
@@ -92,12 +94,15 @@ def setUpClass(cls):
             raise Exception()
 
         cls.mdb = MindsdbNative(config)
+        datastore = DataStore(config)
 
         models = cls.mdb.get_models()
         models = [x['name'] for x in models]
         if TEST_PREDICTOR_NAME in models:
             cls.mdb.delete_model(TEST_PREDICTOR_NAME)
 
+        test_csv_path = get_test_csv(TEST_CSV['name'], TEST_CSV['url'])
+
         query('create database if not exists test')
         test_tables = query('show tables from test')
         test_tables = [x[0] for x in test_tables]
@@ -116,12 +121,6 @@ def setUpClass(cls):
                 )
             ''')
 
-            test_csv_path = str(TESTS_ROOT.joinpath('temp/', TEST_CSV['name']).resolve())
-            if os.path.isfile(test_csv_path) is False:
-                r = requests.get(TEST_CSV['url'])
-                with open(test_csv_path, 'wb') as f:
-                    f.write(r.content)
-
             with open(test_csv_path) as f:
                 csvf = csv.reader(f)
                 i = 0
@@ -148,6 +147,12 @@ def setUpClass(cls):
                     i += 1
             print('done')
 
+        ds = datastore.get_datasource(EXTERNAL_DS_NAME)
+        if ds is not None:
+            datastore.delete_datasource(EXTERNAL_DS_NAME)
+        short_csv_file_path = get_test_csv(f'{EXTERNAL_DS_NAME}.csv', TEST_CSV['url'], lines_count=300, rewrite=True)
+        datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv', short_csv_file_path)
+
     def test_1_initial_state(self):
         print(f'\nExecuting {inspect.stack()[0].function}')
         print('Check all testing objects not exists')
@@ -194,8 +199,8 @@ def test_2_insert_predictor(self):
         mindsdb_tables = [x[0] for x in mindsdb_tables]
         self.assertTrue(TEST_PREDICTOR_NAME in mindsdb_tables)
 
-    def test_3_insert_predictor_with_existing_ds(self):
-        name = f'{TEST_PREDICTOR_NAME}_2'
+    def test_3_externael_ds(self):
+        name = f'{TEST_PREDICTOR_NAME}_external'
         models = self.mdb.get_models()
         models = [x['name'] for x in models]
         if name in models:
@@ -206,7 +211,7 @@ def test_3_insert_predictor_with_existing_ds(self):
             (
                 '{name}',
                 'rental_price, location',
-                '{TEST_PREDICTOR_NAME}',
+                '{EXTERNAL_DS_NAME}',
                 '{{"join_learn_process": true, "stop_training_in_x_seconds": 3}}'
             );
         """)
@@ -221,6 +226,19 @@ def test_3_insert_predictor_with_existing_ds(self):
         mindsdb_tables = [x[0] for x in mindsdb_tables]
         self.assertTrue(name in mindsdb_tables)
 
+        res = query(f"""
+            select
+                rental_price, location, sqft, number_of_rooms,
+                rental_price_confidence, rental_price_min, rental_price_max, rental_price_explain
+            from
+                mindsdb.{name} where external_datasource='{EXTERNAL_DS_NAME}'
+        """, as_dict=True)
+
+        print('check result')
+        self.assertTrue(len(res) > 0)
+        self.assertTrue(res[0]['rental_price'] is not None and res[0]['rental_price'] != 'None')
+        self.assertTrue(res[0]['location'] is not None and res[0]['location'] != 'None')
+
     def test_4_query_predictor(self):
         print(f'\nExecuting {inspect.stack()[0].function}')
         res = query(f"""