Repository: mindsdb/mindsdb

Issue:
Use `mindsdb_sql` to parse `delete` statements



Discussion:

Resolving patch:
diff --git a/mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py b/mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py
--- a/mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py
+++ b/mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py
@@ -1,83 +1,29 @@
 import re
-import json
 
 from pyparsing import (
     CaselessKeyword,
+    originalTextFor,
     ParseException,
     ParserElement,
     QuotedString,
     ZeroOrMore,
-    StringEnd,
-    OneOrMore,
-    Suppress,
-    Optional,
-    Literal,
-    SkipTo,
-    Group,
-    Word,
-    originalTextFor,
-    delimitedList,
-    quotedString,
-    printables,
     nestedExpr,
     restOfLine,
-    alphanums,
-    tokenMap,
-    alphas,
-    nums
-)
-from mindsdb_sql.parser.ast import (
-    Join
+    StringEnd,
+    Literal,
+    SkipTo
 )
-from mindsdb_sql import parse_sql
-
 
 RE_INT = re.compile(r'^[-+]?([1-9]\d*|0)$')
 RE_FLOAT = re.compile(r'^[-+]?([1-9]\d*\.\d*|0\.|0\.\d*)$')
 
 
-class SqlStatementParseError(Exception):
-    pass
-
-
-class SQLParameter:
-    pass
-
-
-SQL_PARAMETER = SQLParameter()
-
-
-class SQLDefault:
-    pass
-
-
-SQL_DEFAULT = SQLDefault()
-
-
 class SqlStatementParser():
-    """Parser for initial analysis of sql statements.
-    Example of usage:
-
-        sql = "insert into a.b (col1) values ('val1')"
-        statement = SqlStatementParser(sql)
-        print(statement.keyword)    # insert
-        print(statement.struct)     # {'database': 'a', 'table': 'b', 'columns': ['col1'], 'values': ['val1']}
-    """
-    _original_sql: str = None
-    _sql: str = None
-    _struct: str = None
-
-    def __init__(self, text, init_parse=True):
+    def __init__(self, text):
         self._original_sql = text
         self._sql = SqlStatementParser.clear_sql(text)
         self._keyword = SqlStatementParser.get_keyword(self._sql)
-        if init_parse:
-            if self._keyword == 'delete':
-                self._struct = self.parse_as_delete()
-            elif self._keyword in 'create_ai_table':
-                self._struct = self.parse_as_create_ai_table()
-            else:
-                self._struct = None
+        self._struct = None
 
     @property
     def keyword(self):
@@ -181,34 +127,6 @@ def get_keyword(sql):
 
         return keyword
 
-    @staticmethod
-    def is_quoted_str(text):
-        if isinstance(text, str) is False:
-            return False
-        for quote in ['"', "'", '`']:
-            if text.startswith(quote) and text.endswith(quote):
-                return True
-        return False
-
-    @staticmethod
-    def is_int_str(text):
-        if isinstance(text, str) and RE_INT.match(text):
-            return True
-        return False
-
-    @staticmethod
-    def is_float_str(text):
-        if isinstance(text, str) and RE_FLOAT.match(text):
-            return True
-        return False
-
-    @staticmethod
-    def unquote(text):
-        for quote in ['"', "'", '`']:
-            if text.startswith(quote) and text.endswith(quote):
-                return text[1:-1]
-        return text
-
     def ends_with(self, text):
         ''' Check if sql ends with 'text'. Not case sensitive.
         '''
@@ -234,325 +152,3 @@ def cut_from_tail(self, text):
 
         self._sql = r.asDict()['original'].strip()
         return True
-
-    def parse_as_create_ai_table(self) -> dict:
-        CREATE, AI, TABLE, VIEW, FROM, USING, AS = map(
-            CaselessKeyword, "CREATE AI TABLE VIEW FROM USING AS".split()
-        )
-
-        AI_TABLE = AI + TABLE
-
-        word = Word(alphanums + "_")
-
-        expr = (
-            CREATE + (AI_TABLE | VIEW) + word('ai_table_name') + AS
-            + originalTextFor(nestedExpr('(', ')'))('select')
-        )
-
-        r = expr.parseString(self._sql)
-        r = r.asDict()
-
-        if r['select'].startswith('(') and r['select'].endswith(')'):
-            r['select'] = r['select'][1:-1]
-        r['select'] = r['select'].strip(' \n')
-
-        select = parse_sql(r['select'])
-
-        if isinstance(select.from_table, Join) is False:
-            raise Exception("'from' must be like: 'from integration.table join predictor'")
-
-        integration_name = select.from_table.left.parts[0]
-        select.from_table.left.parts = select.from_table.left.parts[1:]
-        integration_name_alias = select.from_table.left.alias.parts[0]
-
-        predictor_name = select.from_table.right.parts[0]
-        predictor_name_alias = select.from_table.right.alias.parts[0]
-        select.from_table = select.from_table.left
-
-        query_fields = []
-        predictor_fields = []
-        predictor_fields_targets = []
-
-        integration_sql = str(select)
-
-        for target in select.targets:
-            if target.parts[0] == integration_name_alias:
-                query_fields.append(target.parts[1])
-                predictor_fields_targets.append(target)
-            elif target.parts[0] == predictor_name_alias:
-                predictor_fields.append(target.parts[1])
-        select.targets = predictor_fields_targets
-
-        res = {
-            'ai_table_name': r['ai_table_name'],
-            'integration_name': integration_name,
-            'integration_query': integration_sql,
-            'query_fields': query_fields,
-            'predictor_name': predictor_name,
-            'predictor_fields': predictor_fields
-        }
-
-        return res
-
-    def parse_as_delete(self) -> dict:
-        ''' Parse delete. Example: 'delete from database.table where column_a= 1 and column_b = 2;'
-        '''
-
-        result = {
-            'database': None,
-            'table': None,
-            'where': {}
-        }
-
-        suppressed_word = Word(alphas).suppress()
-        and_ = Literal("and")
-
-        from_value = (
-            QuotedString('`')
-            | originalTextFor(
-                Word(printables, excludeChars='.`')
-            )
-        )
-
-        expr = (
-            suppressed_word + suppressed_word
-            + (delimitedList(from_value, delim='.'))('db_table')
-            + Optional(
-                Word("where").suppress()
-                + OneOrMore(
-                    Word(printables).setResultsName('columns', listAllMatches=True)
-                    + Word('=').suppress()
-                    + Word(printables).setResultsName('values', listAllMatches=True)
-                    + Optional(and_).suppress()
-                )
-            )
-        )
-
-        r = expr.parseString(self._sql).asDict()
-
-        if len(r['db_table']) == 2:
-            result['database'] = r['db_table'][0]
-            result['table'] = r['db_table'][1]
-        else:
-            result['table'] = r['db_table'][0]
-
-        if 'columns' in r and 'values' in r:
-            if not isinstance(r['columns'], list) \
-                    and not isinstance(r['values'], list):
-                r['columns'] = [r['columns']]
-                r['values'] = [r['values']]
-            if len(r['columns']) != len(r['values']):
-                raise SqlStatementParseError("Columns and values have different amounts")
-
-            for i, val in enumerate(r['values']):
-                if isinstance(val, str) and val.lower() == 'null':
-                    result['where'][r['columns'][i]] = None
-                elif val == '?':
-                    result['where'][r['columns'][i]] = SQL_PARAMETER
-                elif isinstance(val, str) and val.lower() == 'default':
-                    result['where'][r['columns'][i]] = SQL_DEFAULT
-                elif SqlStatementParser.is_int_str(val):
-                    result['where'][r['columns'][i]] = int(val)
-                elif SqlStatementParser.is_float_str(val):
-                    result['where'][r['columns'][i]] = float(val)
-                elif SqlStatementParser.is_quoted_str(val):
-                    result['where'][r['columns'][i]] = SqlStatementParser.unquote(val)
-                elif isinstance(val, str):
-                    # it should be in one case, only if server send function as argument, for example:
-                    # insert into table (datetime) values (now())
-                    raise Exception(f"Error: cant determine type of '{val}'")
-
-        for key, value in result['where'].items():
-            if SqlStatementParser.is_quoted_str(value):
-                result['where'][key] = SqlStatementParser.unquote(value)
-
-        return result
-
-    @staticmethod
-    def test_create():
-        def check_recursive(a, b):
-            assert type(a) == type(b)
-            if isinstance(a, dict):
-                for key in a:
-                    check_recursive(a[key], b[key])
-            elif isinstance(a, list):
-                for i in range(len(a)):
-                    check_recursive(a[i], b[i])
-            else:
-                assert a == b
-
-        tests = [[
-            '''
-                CREATE PREDICTor debt_model_1
-                FROM integration_name (select whatever) as ds_name
-                PREDICT f1 as f1_alias, f2, f3 as f3_alias
-                order by f_order_1 ASK, f_order_2, f_order_3 DESC
-                group by f_group_1, f_group_2
-                window 100
-                HORIZON 7
-                using {"x": 1, "y": "a"}
-            ''', {
-                'predictor_name': 'debt_model_1',
-                'integration_name': 'integration_name',
-                'select': 'select whatever',
-                'datasource_name': 'ds_name',
-                'predict': [{'name': 'f1', 'alias': 'f1_alias'}, {'name': 'f2'}, {'name': 'f3', 'alias': 'f3_alias'}],
-                # 'order_by': [{'name': 'f_order_1', 'sort': 'ASK'}, {'name': 'f_order_2'}, {'name': 'f_order_3', 'sort': 'DESC'}],
-                'order_by': ['f_order_1', 'f_order_2', 'f_order_3'],
-                'group_by': ['f_group_1', 'f_group_2'],
-                'window': 100,
-                'horizon': 7,
-                'using': {'x': 1, 'y': 'a'}
-            }
-        ], [
-            '''
-                CREATE PREDICTOR name
-                FROM file.name
-                PREDICT f1
-            ''',
-            {
-                'predictor_name': 'name',
-                'integration_name': 'file.name',
-                'select': None,
-                'predict': [{'name': 'f1'}]
-            }
-        ], [
-            # '''
-            # CREATE AI table ai_table_name
-            # FROM integration (select * from table)
-            # USING model_name
-            # ''',
-            '''
-            CREATE AI table ai_table_name as (
-                SELECT
-                    a.col1,
-                    a.col2,
-                    a.col3,
-                    p.col3 as pred_col3
-                FROM integration_name.table_name as a
-                JOIN predictor_name as p
-            )
-            ''',
-            {
-                'ai_table_name': 'ai_table_name',
-                'integration_name': 'integration_name',
-                'integration_query': 'SELECT a.col1, a.col2, a.col3, p.col3 AS pred_col3 FROM table_name AS a',
-                'query_fields': ['col1', 'col2', 'col3'],
-                'predictor_name': 'predictor_name',
-                'predictor_fields': ['col3']
-            }
-        ]]
-        for sql, result in tests:
-            statement = SqlStatementParser(sql)
-            struct = statement.struct
-            check_recursive(struct, result)
-            check_recursive(result, struct)
-
-    @staticmethod
-    def test():
-        tests = [[
-            'deSCribe predictor',
-            {
-                'keyword': 'describe',
-                'struct': {
-                    'predictor_name': 'predictor'
-                }
-            }
-        ], [
-            'CREATE DATASOURce name FROM mysql WITH {"user": "admin", "password": "password", "host": "127.0.0.1"}',
-            {
-                'keyword': 'create_datasource',
-                'struct': {
-                    'datasource_name': 'name',
-                    'database_type': 'mysql',
-                    'connection_args': {"user": "admin", "password": "password", "host": "127.0.0.1"}
-                }
-            }
-        ], [
-            'retraIN predictor',
-            {
-                'keyword': 'retrain',
-                'struct': {
-                    'predictor_name': 'predictor'
-                }
-            }
-        ], [
-            'retraIN predictor',
-            {
-                'keyword': 'retrain',
-                'struct': {
-                    'predictor_name': 'predictor'
-                }
-            }
-        ], [
-            'start transaction',
-            {'keyword': 'start'}
-        ], [
-            ' START transaction',
-            {'keyword': 'start'}
-        ], [
-            "delete from database_a.table_a where column_a = 1",
-            {
-                'keyword': 'delete',
-                'struct': {
-                    'database': 'database_a',
-                    'table': 'table_a',
-                    'where': {
-                        'column_a': 1
-                    }
-                }
-            }
-        ], [
-            "delete from table_a where column_a = 1 and column_b = ?;",
-            {
-                'keyword': 'delete',
-                'struct': {
-                    'database': None,
-                    'table': 'table_a',
-                    'where': {
-                        'column_a': 1,
-                        'column_b': SQL_PARAMETER
-                    }
-                }
-            }
-        ], [
-            "delete from database_c.table_a where column_a = ? and column_b = ?;",
-            {
-                'keyword': 'delete',
-                'struct': {
-                    'database': 'database_c',
-                    'table': 'table_a',
-                    'where': {
-                        'column_a': SQL_PARAMETER,
-                        'column_b': SQL_PARAMETER
-                    }
-                }
-            }
-        ]]
-        for test in tests:
-            sql = test[0]
-            print(sql)
-            checks = test[1]
-            statement = SqlStatementParser(sql)
-            if 'keyword' in checks:
-                assert(statement.keyword == checks['keyword'])
-            if 'struct' in checks:
-                struct_check = checks['struct']
-                if 'database' in struct_check:
-                    assert(struct_check['database'] == statement.struct['database'])
-                if 'table' in struct_check:
-                    assert(struct_check['table'] == statement.struct['table'])
-                if 'where' in struct_check:
-                    for idx, key in enumerate(struct_check['where'].keys()):
-                        assert(struct_check['where'][key] == statement.struct['where'][key])
-                for key in ['columns', 'values']:
-                    if key in struct_check:
-                        assert(len(struct_check[key]) == len(statement.struct[key]))
-                        for i, el in enumerate(struct_check[key]):
-                            assert(el == statement.struct[key][i])
-        print('tests done!')
-
-
-if __name__ == "__main__":
-    SqlStatementParser.test_create()
-    SqlStatementParser.test()
diff --git a/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/command_packet.py b/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/command_packet.py
--- a/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/command_packet.py
+++ b/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/command_packet.py
@@ -18,7 +18,6 @@
 from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packet import Packet
 from mindsdb.api.mysql.mysql_proxy.data_types.mysql_datum import Datum
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import COMMANDS, getConstName, TYPES
-from mindsdb.api.mysql.mysql_proxy.classes.sql_statement_parser import SQL_PARAMETER
 
 
 class CommandPacket(Packet):
diff --git a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
--- a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
+++ b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
@@ -39,6 +39,7 @@
     Constant,
     Function,
     Explain,
+    Delete,
     Insert,
     Select,
     Star,
@@ -56,7 +57,7 @@
 from mindsdb.api.mysql.mysql_proxy.controllers.session_controller import SessionController
 from mindsdb.api.mysql.mysql_proxy.classes.client_capabilities import ClentCapabilities
 from mindsdb.api.mysql.mysql_proxy.classes.server_capabilities import server_capabilities
-from mindsdb.api.mysql.mysql_proxy.classes.sql_statement_parser import SqlStatementParser, SQL_PARAMETER, SQL_DEFAULT
+from mindsdb.api.mysql.mysql_proxy.classes.sql_statement_parser import SqlStatementParser
 from mindsdb.api.mysql.mysql_proxy.utilities import log
 from mindsdb.api.mysql.mysql_proxy.external_libs.mysql_scramble import scramble as scramble_func
 from mindsdb.api.mysql.mysql_proxy.classes.sql_query import (
@@ -335,10 +336,6 @@ def insert_predictor_answer(self, insert):
         model_interface = self.session.model_interface
         data_store = self.session.data_store
 
-        for key in insert.keys():
-            if insert[key] is SQL_DEFAULT:
-                insert[key] = None  # all default values at this moment is null (None)
-
         select_data_query = insert.get('select_data_query')
         if isinstance(select_data_query, str) is False or len(select_data_query) == 0:
             self.packet(
@@ -691,7 +688,7 @@ def answer_stmt_prepare(self, statement):
             num_params = 0
             for row in statement.values:
                 for item in row:
-                    if isinstance(item, Parameter):
+                    if type(item) == Parameter:
                         num_params = num_params + 1
             num_columns = len(query.columns) - num_params
 
@@ -849,7 +846,7 @@ def answer_stmt_execute(self, stmt_id, parameters):
             parameter_index = 0
             for row in statement.values:
                 for item_index, item in enumerate(row):
-                    if isinstance(item, Parameter):
+                    if type(item) == Parameter:
                         row[item_index] = Constant(parameters[parameter_index])
                         parameter_index += 1
             self.process_insert(statement)
@@ -1118,7 +1115,7 @@ def query_answer(self, sql):
             # not all statemts are parse by parse_sql
             pass
 
-        if isinstance(statement, CreateDatasource):
+        if type(statement) == CreateDatasource:
             struct = {
                 'datasource_name': statement.name,
                 'database_type': statement.engine,
@@ -1126,7 +1123,7 @@ def query_answer(self, sql):
             }
             self.answer_create_datasource(struct)
             return
-        if isinstance(statement, DropPredictor):
+        if type(statement) == DropPredictor:
             predictor_name = statement.name.parts[-1]
             self.session.datahub['mindsdb'].delete_predictor(predictor_name)
             self.packet(OkPacket).send()
@@ -1134,17 +1131,17 @@ def query_answer(self, sql):
             # fallback for statement
             self.answer_create_datasource(struct)
             return
-        elif isinstance(statement, DropDatasource):
+        elif type(statement) == DropDatasource:
             ds_name = statement.name.parts[-1]
             self.answer_drop_datasource(ds_name)
             return
-        elif isinstance(statement, Describe):
+        elif type(statement) == Describe:
             self.answer_describe_predictor(statement.value.parts[-1])
             return
         elif type(statement) == RetrainPredictor:
             self.answer_retrain_predictor(statement.name.parts[-1])
             return
-        elif isinstance(statement, Show) or keyword == 'show':
+        elif type(statement) == Show or keyword == 'show':
             sql_category = statement.category.lower()
             condition = statement.condition.lower() if isinstance(statement.condition, str) else statement.condition
             expression = statement.expression
@@ -1306,9 +1303,9 @@ def query_answer(self, sql):
             elif sql_category in ('character set', 'charset'):
                 charset = None
                 if condition == 'where':
-                    if isinstance(expression, BinaryOperation):
+                    if type(expression) == BinaryOperation:
                         if expression.op == '=':
-                            if isinstance(expression.args[0], Identifier):
+                            if type(expression.args[0]) == Identifier:
                                 if expression.args[0].parts[0].lower() == 'charset':
                                     charset = expression.args[1].value
                                 else:
@@ -1358,9 +1355,9 @@ def query_answer(self, sql):
             elif sql_category == 'table status':
                 # SHOW TABLE STATUS LIKE 'table'
                 table_name = None
-                if condition == 'like' and isinstance(expression, Constant):
+                if condition == 'like' and type(expression) == Constant:
                     table_name = expression.value
-                elif condition == 'from' and isinstance(expression, Identifier):
+                elif condition == 'from' and type(expression) == Identifier:
                     table_name = expression.parts[-1]
                 if table_name is None:
                     err_str = f"Can't determine table name in query: {sql}"
@@ -1370,11 +1367,11 @@ def query_answer(self, sql):
                 return
             else:
                 raise Exception(f'Statement not implemented: {sql}')
-        elif isinstance(statement, (StartTransaction, CommitTransaction, RollbackTransaction)):
+        elif type(statement) in (StartTransaction, CommitTransaction, RollbackTransaction):
             self.packet(OkPacket).send()
-        elif isinstance(statement, Set):
+        elif type(statement) == Set:
             category = (statement.category or '').lower()
-            if category == '' and isinstance(statement.arg, BinaryOperation):
+            if category == '' and type(statement.arg) == BinaryOperation:
                 self.packet(OkPacket).send()
             elif category == 'autocommit':
                 self.packet(OkPacket).send()
@@ -1400,28 +1397,31 @@ def query_answer(self, sql):
             else:
                 log.warning(f'SQL statement is not processable, return OK package: {sql}')
                 self.packet(OkPacket).send()
-        elif isinstance(statement, Use):
+        elif type(statement) == Use:
             db_name = statement.value.parts[-1]
             self.change_default_db(db_name)
             self.packet(OkPacket).send()
-        elif isinstance(statement, CreatePredictor):
+        elif type(statement) == CreatePredictor:
             self.answer_create_predictor(statement)
         elif keyword == 'set':
             log.warning(f'Unknown SET query, return OK package: {sql}')
             self.packet(OkPacket).send()
         elif keyword == 'create_ai_table':
             self.answer_create_ai_table(struct)
-        elif keyword == 'delete' and \
-                ('mindsdb.predictors' in sql_lower or self.session.database == 'mindsdb' and 'predictors' in sql_lower):
-            self.delete_predictor_sql(sql)
+        elif type(statement) == Delete:
+            if self.session.database != 'mindsdb' and statement.table.parts[0] != 'mindsdb':
+                raise Exception("Only 'DELETE' from database 'mindsdb' is possible at this moment")
+            if statement.table.parts[-1] != 'predictors':
+                raise Exception("Only 'DELETE' from table 'mindsdb.predictors' is possible at this moment")
+            self.delete_predictor_sql(str(sql))
             self.packet(OkPacket).send()
-        elif isinstance(statement, Insert):
+        elif type(statement) == Insert:
             self.process_insert(statement)
         elif keyword in ('update', 'insert'):
             raise NotImplementedError('Update and Insert are not implemented')
         elif keyword == 'alter' and ('disable keys' in sql_lower) or ('enable keys' in sql_lower):
             self.packet(OkPacket).send()
-        elif isinstance(statement, Select):
+        elif type(statement) == Select:
             if statement.from_table is None:
                 self.answer_single_row_select(statement)
                 return
@@ -1466,7 +1466,7 @@ def query_answer(self, sql):
                 session=self.session
             )
             self.answer_select(query)
-        elif isinstance(statement, Explain):
+        elif type(statement) == Explain:
             self.answer_explain_table(statement.target.parts)
         else:
             print(sql)
@@ -1477,7 +1477,7 @@ def answer_single_row_select(self, statement):
         columns = []
         data = []
         for target in statement.targets:
-            if isinstance(target, Variable):
+            if type(target) == Variable:
                 var_name = target.value
                 column_name = f'@@{var_name}'
                 column_alias = target.alias or column_name
@@ -1487,7 +1487,7 @@ def answer_single_row_select(self, statement):
                     result = ''
                 else:
                     result = result[0]
-            elif isinstance(target, Function):
+            elif type(target) == Function:
                 functions_results = {
                     'connection_id': self.connection_id,
                     'database': self.session.database,
@@ -1499,14 +1499,14 @@ def answer_single_row_select(self, statement):
                 column_name = f'{target.op}()'
                 column_alias = target.alias or column_name
                 result = functions_results[function_name]
-            elif isinstance(target, Constant):
+            elif type(target) == Constant:
                 result = target.value
                 column_name = str(result)
-                column_alias = '.'.join(target.alias.parts) if isinstance(target.alias, Identifier) else column_name
-            elif isinstance(target, Identifier):
+                column_alias = '.'.join(target.alias.parts) if type(target.alias) == Identifier else column_name
+            elif type(target) == Identifier:
                 result = '.'.join(target.parts)
                 column_name = str(result)
-                column_alias = '.'.join(target.alias.parts) if isinstance(target.alias, Identifier) else column_name
+                column_alias = '.'.join(target.alias.parts) if type(target.alias) == Identifier else column_name
 
             columns.append({
                 'table_name': '',


Test patch: