Repository: mindsdb/mindsdb

Issue:
Datastore dir created when error happens
The user can include wrong parameters or have a typo in INSERT OR SELECT query which will result in an error but the directory inside datastore will be created, so the second time user will get `[Errno 17] File exists: '/mindsdb-examples/tescl/datastore/airq_predictor'`. 


Discussion:

Resolving patch:
diff --git a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
--- a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
+++ b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
@@ -363,12 +363,20 @@ def insert_predictor_answer(self, insert):
                 ).send()
                 return
             insert['select_data_query'] = insert['select_data_query'].replace(r"\'", "'")
-            ds = default_store.save_datasource(insert['name'], integration, insert['select_data_query'])
+            ds, ds_name = default_store.save_datasource(insert['name'], integration, insert['select_data_query'])
         elif is_external_datasource:
             ds = default_store.get_datasource_obj(insert['external_datasource'], raw=True)
+            ds_name = insert['external_datasource']
 
         insert['predict'] = [x.strip() for x in insert['predict'].split(',')]
 
+        ds_columns = [x['name'] for x in default_store.get_datasource(ds_name)['columns']]
+        for col in insert['predict']:
+            if col not in ds_columns:
+                if is_select_data_query:
+                    default_store.delete_datasource(ds_name)
+                raise Exception(f"Column '{col}' not exists")
+
         mdb.learn(insert['name'], ds, insert['predict'], kwargs)
 
         self.packet(OkPacket).send()
@@ -779,7 +787,7 @@ def queryAnswer(self, sql):
             else:
                 self.packet(OkPacket).send()
         elif keyword == 'use':
-            self.session.database = sql_lower.split()[1].trim(' ;')
+            self.session.database = sql_lower.split()[1].strip(' ;')
             self.packet(OkPacket).send()
         elif 'show warnings' in sql_lower:
             self.answerShowWarnings()
diff --git a/mindsdb/integrations/clickhouse/clickhouse.py b/mindsdb/integrations/clickhouse/clickhouse.py
--- a/mindsdb/integrations/clickhouse/clickhouse.py
+++ b/mindsdb/integrations/clickhouse/clickhouse.py
@@ -21,7 +21,7 @@ def _to_clickhouse_table(self, stats, predicted_cols):
             DATA_SUBTYPES.AUDIO: 'Nullable(String)',
             DATA_SUBTYPES.SHORT: 'Nullable(String)',
             DATA_SUBTYPES.RICH: 'Nullable(String)',
-            DATA_SUBTYPES.ARRAY: 'Array(Float64)'
+            DATA_SUBTYPES.ARRAY: 'Nullable(String)'
         }
 
         column_declaration = []
diff --git a/mindsdb/interfaces/datastore/datastore.py b/mindsdb/interfaces/datastore/datastore.py
--- a/mindsdb/interfaces/datastore/datastore.py
+++ b/mindsdb/interfaces/datastore/datastore.py
@@ -4,17 +4,12 @@
 import shutil
 import os
 import pickle
-import sys
 
-if os.name == 'posix':
-    import resource
-
-import mindsdb
-
-from mindsdb.interfaces.datastore.sqlite_helpers import *
+from mindsdb.interfaces.datastore.sqlite_helpers import get_sqlite_data, cast_df_columns_types, create_sqlite_db
 from mindsdb.interfaces.native.mindsdb import MindsdbNative
 from mindsdb_native import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS
 
+
 class DataStore():
     def __init__(self, config, storage_dir=None):
         self.config = config
@@ -22,12 +17,11 @@ def __init__(self, config, storage_dir=None):
         self.mindsdb_native = MindsdbNative(config)
 
     def get_analysis(self, ds):
-        if isinstance(ds,str):
+        if isinstance(ds, str):
             return self.mindsdb_native.analyse_dataset(self.get_datasource_obj(ds))
         else:
             return self.mindsdb_native.analyse_dataset(ds)
 
-
     def get_datasources(self):
         datasource_arr = []
         for ds_name in os.listdir(self.dir):
@@ -77,13 +71,18 @@ def save_datasource(self, name, source_type, source, file_path=None):
         os.mkdir(ds_dir)
 
         if source_type == 'file':
-            source = os.path.join(ds_dir, source)
-            os.replace(file_path, source)
-            ds = FileDS(source)
+            try:
+                source = os.path.join(ds_dir, source)
+                os.replace(file_path, source)
+                ds = FileDS(source)
+            except Exception:
+                shutil.rmtree(ds_meta_dir)
+                raise
+
             picklable = {
-                'class': 'FileDS'
-                ,'args': [source]
-                ,'kwargs': {}
+                'class': 'FileDS',
+                'args': [source],
+                'kwargs': {}
             }
         elif source_type in self.config['integrations']:
             integration = self.config['integrations'][source_type]
@@ -112,21 +111,29 @@ def save_datasource(self, name, source_type, source, file_path=None):
                 picklable['class'] = 'PostgresDS'
             else:
                 raise ValueError(f'Unknown DS source_type: {source_type}')
-            ds = dsClass(
-                query=source,
-                user=integration['user'],
-                password=integration['password'],
-                host=integration['host'],
-                port=integration['port']
-            )
+            try:
+                ds = dsClass(
+                    query=source,
+                    user=integration['user'],
+                    password=integration['password'],
+                    host=integration['host'],
+                    port=integration['port']
+                )
+            except Exception:
+                shutil.rmtree(ds_meta_dir)
+                raise
         else:
             # This probably only happens for urls
             print('Create URL data source !')
-            ds = FileDS(source)
+            try:
+                ds = FileDS(source)
+            except Exception:
+                shutil.rmtree(ds_meta_dir)
+                raise
             picklable = {
-                'class': 'FileDS'
-                ,'args': [source]
-                ,'kwargs': {}
+                'class': 'FileDS',
+                'args': [source],
+                'kwargs': {}
             }
 
         df = ds.df
@@ -134,10 +141,10 @@ def save_datasource(self, name, source_type, source, file_path=None):
         df_with_types = cast_df_columns_types(df, self.get_analysis(df)['data_analysis_v2'])
         create_sqlite_db(os.path.join(ds_dir, 'sqlite.db'), df_with_types)
 
-        with open(os.path.join(ds_dir,'ds.pickle'), 'wb') as fp:
+        with open(os.path.join(ds_dir, 'ds.pickle'), 'wb') as fp:
             pickle.dump(picklable, fp)
 
-        with open(os.path.join(ds_dir,'metadata.json'), 'w') as fp:
+        with open(os.path.join(ds_dir, 'metadata.json'), 'w') as fp:
             meta = {
                 'name': name,
                 'source_type': source_type,
@@ -149,24 +156,21 @@ def save_datasource(self, name, source_type, source, file_path=None):
             }
             json.dump(meta, fp)
 
-        return self.get_datasource_obj(name, raw=True)
+        return self.get_datasource_obj(name, raw=True), name
 
     def get_datasource_obj(self, name, raw=False):
         ds_meta_dir = os.path.join(self.dir, name)
         ds_dir = os.path.join(ds_meta_dir, 'datasource')
         ds = None
         try:
-            #resource.setrlimit(resource.RLIMIT_STACK, [0x10000000, resource.RLIM_INFINITY])
-            #sys.setrecursionlimit(0x100000)
-            with open(os.path.join(ds_dir,'ds.pickle'), 'rb') as fp:
+            with open(os.path.join(ds_dir, 'ds.pickle'), 'rb') as fp:
                 picklable = pickle.load(fp)
                 if raw:
                     return picklable
                 try:
-                    ds = eval(picklable['class'])(*picklable['args'],**picklable['kwargs'])
-                except:
+                    ds = eval(picklable['class'])(*picklable['args'], **picklable['kwargs'])
+                except Exception:
                     ds = picklable
-
             return ds
         except Exception as e:
             print(f'\n{e}\n')


Test patch: