Repository: mindsdb/mindsdb

Issue:
Remove `columns` and other datasource metadata required by sqlite 
Title says it all, start work on it once https://github.com/mindsdb/mindsdb/issues/907 is done.



Discussion:

Resolving patch:
diff --git a/mindsdb/api/http/namespaces/datasource.py b/mindsdb/api/http/namespaces/datasource.py
--- a/mindsdb/api/http/namespaces/datasource.py
+++ b/mindsdb/api/http/namespaces/datasource.py
@@ -2,7 +2,7 @@
 import os
 import threading
 import tempfile
-
+import re
 import multipart
 
 import mindsdb
@@ -11,7 +11,6 @@
 from flask_restx import Resource, abort     # 'abort' using to return errors as json: {'message': 'error text'}
 from flask import current_app as ca
 
-from mindsdb.interfaces.datastore.sqlite_helpers import *
 from mindsdb.api.http.namespaces.configs.datasources import ns_conf
 from mindsdb.api.http.namespaces.entitites.datasources.datasource import (
     datasource_metadata,
@@ -30,6 +29,27 @@
 )
 
 
+def parse_filter(key, value):
+    result = re.search(r'filter(_*.*)\[(.*)\]', key)
+    operator = result.groups()[0].strip('_') or 'like'
+    field = result.groups()[1]
+    operators_map = {
+        'like': 'like',
+        'in': 'in',
+        'nin': 'not in',
+        'gt': '>',
+        'lt': '<',
+        'gte': '>=',
+        'lte': '<=',
+        'eq': '=',
+        'neq': '!='
+    }
+    if operator not in operators_map:
+        return None
+    operator = operators_map[operator]
+    return [field, operator, value]
+
+
 @ns_conf.route('/')
 class DatasourcesList(Resource):
     @ns_conf.doc('get_datasources_list')
@@ -68,7 +88,6 @@ def put(self, name):
         data = {}
 
         def on_field(field):
-            print(f'\n\n{field}\n\n')
             name = field.field_name.decode()
             value = field.value.decode()
             data[name] = value
diff --git a/mindsdb/interfaces/datastore/datastore.py b/mindsdb/interfaces/datastore/datastore.py
--- a/mindsdb/interfaces/datastore/datastore.py
+++ b/mindsdb/interfaces/datastore/datastore.py
@@ -5,12 +5,10 @@
 import os
 import pickle
 
-from mindsdb.interfaces.datastore.sqlite_helpers import get_sqlite_data, cast_df_columns_types, create_sqlite_db
 from mindsdb.interfaces.native.mindsdb import MindsdbNative
 from mindsdb_native import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS
 
 
-
 class DataStore():
     def __init__(self, config):
         self.config = config
@@ -40,8 +38,25 @@ def get_datasources(self):
         return datasource_arr
 
     def get_data(self, name, where=None, limit=None, offset=None):
-        # @TODO Apply filter directly to postgres/mysql/clickhouse/etc...  when the datasource is of that type
-        return get_sqlite_data(os.path.join(self.dir, name, 'sqlite.db'), where=where, limit=limit, offset=offset)
+        if offset is None:
+            offset = 0
+
+        ds = self.get_datasource_obj(name)
+
+        # @TODO Remove and add `offset` to the `filter` method of the datasource
+        if limit is not None:
+            filtered_ds = ds.filter(where=where, limit=limit+offset)
+        else:
+            filtered_ds = ds.filter(where=where)
+
+        filtered_ds = filtered_ds.iloc[offset:]
+
+        data = filtered_ds.to_dict(orient='records')
+        return {
+            'data': data,
+            'rowcount': len(ds),
+            'columns_names': filtered_ds.columns
+        }
 
     def get_datasource(self, name):
         for ds in self.get_datasources():
@@ -168,7 +183,6 @@ def save_datasource(self, name, source_type, source, file_path=None):
                     ds = dsClass(**picklable['kwargs'])
             else:
                 # This probably only happens for urls
-                print('Create URL data source !')
                 ds = FileDS(source)
                 picklable = {
                     'class': 'FileDS',
@@ -182,9 +196,8 @@ def save_datasource(self, name, source_type, source, file_path=None):
                 shutil.rmtree(ds_meta_dir)
                 raise Exception('Each column in datasource must have unique name')
 
-            # limit=200 for analysis might have odd results but it's a compromise that should work most of the time to make the sqlite db creation faster
-            df_with_types = cast_df_columns_types(df, self.get_analysis(ds.filter(limit=200))['data_analysis_v2'])
-            create_sqlite_db(os.path.join(ds_meta_dir, 'sqlite.db'), df_with_types)
+            # Not sure if needed
+            #summary_analysis = self.get_analysis(ds.filter(limit=200))['data_analysis_v2']
 
             with open(os.path.join(ds_meta_dir, 'ds.pickle'), 'wb') as fp:
                 pickle.dump(picklable, fp)
diff --git a/mindsdb/interfaces/datastore/sqlite_helpers.py b/mindsdb/interfaces/datastore/sqlite_helpers.py
deleted file mode 100644
--- a/mindsdb/interfaces/datastore/sqlite_helpers.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import sqlite3
-from mindsdb_native.libs.constants.mindsdb import DATA_TYPES, DATA_SUBTYPES
-import re
-
-
-def create_sqlite_db(path, data_frame):
-    con = sqlite3.connect(path)
-    data_frame.to_sql(name='data', con=con, index=False)
-    con.close()
-
-
-def cast_df_columns_types(df, stats):
-    types_map = {
-        DATA_TYPES.NUMERIC: {
-            DATA_SUBTYPES.INT: 'int64',
-            DATA_SUBTYPES.FLOAT: 'float64',
-            DATA_SUBTYPES.BINARY: 'bool'
-        },
-        DATA_TYPES.DATE: {
-            DATA_SUBTYPES.DATE: 'datetime64',       # YYYY-MM-DD
-            DATA_SUBTYPES.TIMESTAMP: 'datetime64'   # YYYY-MM-DD hh:mm:ss or 1852362464
-        },
-        DATA_TYPES.CATEGORICAL: {
-            DATA_SUBTYPES.SINGLE: 'category',
-            DATA_SUBTYPES.MULTIPLE: 'category'
-        },
-        DATA_TYPES.FILE_PATH: {
-            DATA_SUBTYPES.IMAGE: 'object',
-            DATA_SUBTYPES.VIDEO: 'object',
-            DATA_SUBTYPES.AUDIO: 'object'
-        },
-        DATA_TYPES.SEQUENTIAL: {
-            DATA_SUBTYPES.ARRAY: 'object'
-        },
-        DATA_TYPES.TEXT: {
-            DATA_SUBTYPES.SHORT: 'object',
-            DATA_SUBTYPES.RICH: 'object'
-        }
-    }
-
-    columns = [dict(name=x) for x in list(df.keys())]
-
-    for column in columns:
-        try:
-            name = column['name']
-            col_type = stats[name]['typing']['data_type']
-            col_subtype = stats[name]['typing']['data_subtype']
-            new_type = types_map[col_type][col_subtype]
-            if new_type == 'int64' or new_type == 'float64':
-                df[name] = df[name].apply(lambda x: x.replace(',', '.') if isinstance(x, str) else x)
-            if new_type == 'int64':
-                df = df.astype({name: 'float64'})
-            df = df.astype({name: new_type})
-        except Exception as e:
-            print(e)
-            print(f'Error: cant convert type of DS column {name} to {new_type}')
-
-    return df
-
-
-def parse_filter(key, value):
-    result = re.search(r'filter(_*.*)\[(.*)\]', key)
-    operator = result.groups()[0].strip('_') or 'like'
-    field = result.groups()[1]
-    operators_map = {
-        'like': 'like',
-        'in': 'in',
-        'nin': 'not in',
-        'gt': '>',
-        'lt': '<',
-        'gte': '>=',
-        'lte': '<=',
-        'eq': '=',
-        'neq': '!='
-    }
-    if operator not in operators_map:
-        return None
-    operator = operators_map[operator]
-    return {'field': field, 'value': value, 'operator': operator}
-
-
-def prepare_sql_where(where):
-    marks = {}
-    if len(where) > 0:
-        for i in range(len(where)):
-            field = where[i]['field'].replace('"', '""')
-            operator = where[i]['operator']
-            value = where[i]['value']
-            var_name = f'var{i}'
-            if ' ' in field:
-                field = f'"{field}"'
-            if operator == 'like':
-                marks[var_name] = '%' + value + '%'
-            else:
-                marks[var_name] = value
-            where[i] = f'{field} {operator} :var{i}'
-        where = 'where ' + ' and '.join(where)
-    else:
-        where = ''
-    return where, marks
-
-
-def get_sqlite_columns_names(cursor):
-    cursor.execute('pragma table_info(data);')
-    column_name_index = [x[0] for x in cursor.description].index('name')
-    columns = cursor.fetchall()
-    return [x[column_name_index] for x in columns]
-
-
-def get_sqlite_data(db_path, where, limit, offset):
-    where = [] if where is None else where
-
-    con = sqlite3.connect(db_path)
-    cur = con.cursor()
-
-    offset = '' if limit is None or offset is None else f'offset {offset}'
-    limit = '' if limit is None else f'limit {limit}'
-
-    columns_names = get_sqlite_columns_names(cur)
-    where = [x for x in where if x['field'] in columns_names]
-    where, marks = prepare_sql_where(where)
-
-    count_query = ' '.join(['select count(1) from data', where])
-    cur.execute(count_query, marks)
-    rowcount = cur.fetchone()[0]
-
-    query = ' '.join(['select * from data', where, limit, offset])
-    cur.execute(query, marks)
-    data = cur.fetchall()
-    data = [dict(zip(columns_names, x)) for x in data]
-
-    cur.close()
-    con.close()
-
-    return {
-        'data': data,
-        'rowcount': rowcount,
-        'columns_names': columns_names
-    }


Test patch: