Repository: mindsdb/mindsdb

Issue:
Hive (Hive Server2?) as a Datasource
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Is your feature request related to a problem? Please describe.

A user would like to use Hive as a datasource.



### Describe the solution you'd like.

Be able to train ML models in MindsDB from Hive (server2?)

### Describe an alternate solution.

_No response_

### Anything else? (Additional Context)

This is a feature that a potential paid customer would like.

_We use airbyte to sync data between a Postgres database and a Databricks Spark cluster. We then use DBT to transform those tables into useful data products within Databricks._



Discussion:
Hi @ZoranPandovski. Can I take this up?
Hey sure @bispaul 
Hi @bispaul, did you start on this?
Hi @ZoranPandovski , I have started but couldn't work much due to my workload. I will try to complete this before the end of this month. Thank You.

Resolving patch:
diff --git a/mindsdb/integrations/handlers/hive_handler/__about__.py b/mindsdb/integrations/handlers/hive_handler/__about__.py
new file mode 100644
--- /dev/null
+++ b/mindsdb/integrations/handlers/hive_handler/__about__.py
@@ -0,0 +1,9 @@
+__title__ = 'MindsDB Hive handler'
+__package_name__ = 'mindsdb_hive_handler'
+__version__ = '0.0.1'
+__description__ = "MindsDB handler for Hive2"
+__author__ = 'Biswadip Paul'
+__github__ = 'https://github.com/mindsdb/mindsdb'
+__pypi__ = 'https://pypi.org/project/mindsdb/'
+__license__ = 'GPL-3.0'
+__copyright__ = 'Copyright 2022- mindsdb'
diff --git a/mindsdb/integrations/handlers/hive_handler/__init__.py b/mindsdb/integrations/handlers/hive_handler/__init__.py
new file mode 100644
--- /dev/null
+++ b/mindsdb/integrations/handlers/hive_handler/__init__.py
@@ -0,0 +1,23 @@
+from mindsdb.integrations.libs.const import HANDLER_TYPE
+
+from .__about__ import __version__ as version, __description__ as description
+try:
+    from .hive_handler import (
+        HiveHandler as Handler,
+        connection_args_example,
+        connection_args
+    )
+    import_error = None
+except Exception as e:
+    Handler = None
+    import_error = e
+
+title = 'Hive'
+name = 'hive'
+type = HANDLER_TYPE.DATA
+icon_path = 'icon.svg'
+
+__all__ = [
+    'Handler', 'version', 'name', 'type', 'title', 'description',
+    'connection_args', 'connection_args_example', 'import_error', 'icon_path'
+]
diff --git a/mindsdb/integrations/handlers/hive_handler/hive_handler.py b/mindsdb/integrations/handlers/hive_handler/hive_handler.py
new file mode 100644
--- /dev/null
+++ b/mindsdb/integrations/handlers/hive_handler/hive_handler.py
@@ -0,0 +1,191 @@
+from typing import Optional
+from collections import OrderedDict
+import pandas as pd
+from pyhive import (hive, sqlalchemy_hive)
+from sqlalchemy import create_engine
+
+from mindsdb_sql import parse_sql
+from mindsdb_sql.render.sqlalchemy_render import SqlalchemyRender
+from mindsdb_sql.parser.ast.base import ASTNode
+
+from mindsdb.utilities.log import log
+from mindsdb.integrations.libs.base_handler import DatabaseHandler
+from mindsdb.integrations.libs.response import (
+    HandlerStatusResponse as StatusResponse,
+    HandlerResponse as Response,
+    RESPONSE_TYPE
+)
+from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE
+
+
+class HiveHandler(DatabaseHandler):
+    """
+    This handler handles connection and execution of the Hive SQL statements.
+    """
+
+    name = 'hive'
+
+    def __init__(self, name: str, connection_data: Optional[dict], **kwargs):
+        super().__init__(name)
+        self.parser = parse_sql
+        self.dialect = 'hive'
+
+        self.connection_data = connection_data
+        self.kwargs = kwargs
+
+        self.connection = None
+        self.is_connected = False
+
+    def __del__(self):
+        if self.is_connected is True:
+            self.disconnect()
+
+    def connect(self):
+        if self.is_connected is True:
+            return self.connection
+
+        config = {
+            'host': self.connection_data.get('host'),
+            'port': self.connection_data.get('port'),
+            'auth': self.connection_data.get('auth', 'CUSTOM'),
+            'username': self.connection_data.get('user'),
+            'password': self.connection_data.get('password'),
+            'database': self.connection_data.get('database')
+        }
+
+        connection = hive.Connection(**config)
+        self.is_connected = True
+        self.connection = connection
+        return self.connection
+
+    def disconnect(self):
+        if self.is_connected is False:
+            return
+        self.connection.close()
+        self.is_connected = False
+        return
+
+    def check_connection(self) -> StatusResponse:
+        """
+        Check the connection of the Hive database
+        :return: success status and error message if error occurs
+        """
+
+        response = StatusResponse(False)
+        need_to_close = self.is_connected is False
+
+        try:
+            self.connect()
+            response.success = True
+        except Exception as e:
+            log.error(f'Error connecting to Hive {self.connection_data["database"]}, {e}!')
+            response.error_message = str(e)
+
+        if response.success is True and need_to_close:
+            self.disconnect()
+        if response.success is False and self.is_connected is True:
+            self.is_connected = False
+
+        return response
+
+    def native_query(self, query: str) -> Response:
+        """
+        Receive SQL query and runs it
+        :param query: The SQL query to run in Hive
+        :return: returns the records from the current recordset
+        """
+
+        need_to_close = self.is_connected is False
+
+        connection = self.connect()
+        with connection.cursor() as cur:
+            try:
+                cur.execute(query)
+                result = cur.fetchall()
+                if result:
+                    response = Response(
+                        RESPONSE_TYPE.TABLE,
+                        pd.DataFrame(
+                            result,
+                            columns=[x[0].split('.')[-1] for x in cur.description]
+                        )
+                    )
+                else:
+                    response = Response(RESPONSE_TYPE.OK)
+                connection.commit()
+            except Exception as e:
+                log.error(f'Error running query: {query} on {self.connection_data["database"]}!')
+                response = Response(
+                    RESPONSE_TYPE.ERROR,
+                    error_message=str(e)
+                )
+                connection.rollback()
+
+        if need_to_close is True:
+            self.disconnect()
+
+        return response
+
+    def query(self, query: ASTNode) -> Response:
+        """
+        Retrieve the data from the SQL statement.
+        """
+        renderer = SqlalchemyRender(sqlalchemy_hive.HiveDialect)
+        query_str = renderer.get_string(query, with_failback=True)
+        return self.native_query(query_str)
+        # return self.native_query(query.to_string())
+
+    def get_tables(self) -> Response:
+        """
+        Get a list with all of the tables in Hive
+        """
+        q = "SHOW TABLES"
+        result = self.native_query(q)
+        df = result.data_frame
+        result.data_frame = df.rename(columns={df.columns[0]: 'table_name'})
+        return result
+
+    def get_columns(self, table_name) -> Response:
+        """
+        Show details about the table
+        """
+        q = f"DESCRIBE {table_name}"
+        result = self.native_query(q)
+        return result
+
+
+connection_args = OrderedDict(
+    user={
+        'type': ARG_TYPE.STR,
+        'description': 'The user name used to authenticate with the Hive server.'
+    },
+    password={
+        'type': ARG_TYPE.STR,
+        'description': 'The password to authenticate the user with the Hive server.'
+    },
+    database={
+        'type': ARG_TYPE.STR,
+        'description': 'The database name to use when connecting with the Hive server.'
+    },
+    host={
+        'type': ARG_TYPE.STR,
+        'description': 'The host name or IP address of the Hive server. NOTE: use \'127.0.0.1\' instead of \'localhost\' to connect to local server.'
+    },
+    port={
+        'type': ARG_TYPE.INT,
+        'description': 'The TCP/IP port of the Hive server. Must be an integer.'
+    },
+    auth={
+        'type': ARG_TYPE.STR,
+        'description': 'The Auth type of the Hive server.'
+    }
+)
+
+connection_args_example = OrderedDict(
+    host='127.0.0.1',
+    port='10000',
+    auth='CUSTOM',
+    user='root',
+    password='password',
+    database='database'
+)


Test patch:
diff --git a/mindsdb/integrations/handlers/hive_handler/tests/__init__.py b/mindsdb/integrations/handlers/hive_handler/tests/__init__.py
new file mode 100644
diff --git a/mindsdb/integrations/handlers/hive_handler/tests/test_hive_handler.py b/mindsdb/integrations/handlers/hive_handler/tests/test_hive_handler.py
new file mode 100644
--- /dev/null
+++ b/mindsdb/integrations/handlers/hive_handler/tests/test_hive_handler.py
@@ -0,0 +1,53 @@
+import unittest
+
+from mindsdb.integrations.handlers.hive_handler.hive_handler import HiveHandler
+from mindsdb.api.mysql.mysql_proxy.libs.constants.response_type import RESPONSE_TYPE
+
+
+class HiveHandlerTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.kwargs = {
+            "host": "localhost",
+            "port": "10000",
+            "user": "admin",
+            "password": "password",
+            "database": "default",
+            "auth": "CUSTOM"
+        }
+        cls.handler = HiveHandler('test_hive_handler', **cls.kwargs)
+
+    def test_0_connect(self):
+        self.handler.check_connection()
+
+    def test_1_native_query_show_dbs(self):
+        dbs = self.handler.native_query("SHOW DATABASES;")
+        assert dbs['type'] is not RESPONSE_TYPE.ERROR
+
+    def test_2_get_tables(self):
+        tbls = self.handler.get_tables()
+        assert tbls['type'] is not RESPONSE_TYPE.ERROR
+
+    def test_3_get_views(self):
+        views = self.handler.get_views()
+        assert views['type'] is not RESPONSE_TYPE.ERROR
+
+    def test_5_drop_table(self):
+        res = self.handler.native_query("DROP TABLE IF EXISTS test_hdb")
+        assert res['type'] is not RESPONSE_TYPE.ERROR
+
+    def test_4_create_table(self):
+        res = self.handler.native_query("CREATE TABLE IF NOT EXISTS test_hdb (test_col INT)")
+        assert res['type'] is not RESPONSE_TYPE.ERROR
+
+    def test_6_describe_table(self):
+        described = self.handler.get_columns("test_hdb")
+        assert described['type'] is RESPONSE_TYPE.TABLE
+
+    def test_7_select_query(self):
+        query = "SELECT * FROM test_mdb WHERE foo=238"
+        result = self.handler.query(query)
+        assert result['type'] is RESPONSE_TYPE.TABLE
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file