Repository: ibis-project/ibis

Issue:
bug(impala): cannot connect to Impala backend after 8.0 update due to attempt to create a temporary database on connect
### What happened?

Hello, back in January I reported a bug under a closed merge request, which probably was not noticed. I probably should have opened a separate issue already back then, sorry! The issue seems to be that Ibis is now trying to create a database at startup, which does not work for me, and likely most other corporate users.

### What version of ibis are you using?

8.0

### What backend(s) are you using, if any?

Impala

### Relevant log output

```sh
Please find it here, the error is still the same:

https://github.com/ibis-project/ibis/pull/7537#issuecomment-1876533010


---------------------------------------------------------------------------
HiveServer2Error                          Traceback (most recent call last)
c:\Users\username\Desktop\Projects\projname\src\pairing.py in line 1
----> 281 prod_mmsrdl = connect_ibis(schema_name="schema_name", env="prod")

File ~\Desktop\Projects\projname\src\utils.py:35, in connect_ibis(schema_name, env)
---> 35 return ibis.impala.connect(
     36     host=hosts[env],
     37     database=schema_name,
     38     port=443,
     39     use_ssl=True,
     40     auth_mechanism="PLAIN",
     41     user=os.getlogin(),
     42     password=pw,
     43     use_http_transport=True,
     44     http_path="cliservice",
     45 )

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\__init__.py:107, in __getattr__.<locals>.connect(*args, **kwargs)
    106 def connect(*args, **kwargs):
--> 107     return backend.connect(*args, **kwargs)

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\base\__init__.py:847, in BaseBackend.connect(self, *args, **kwargs)
    824 """Connect to the database.
    825 
    826 Parameters
   (...)
    844     An instance of the backend
    845 """
    846 new_backend = self.__class__(*args, **kwargs)
--> 847 new_backend.reconnect()
    848 return new_backend

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\base\__init__.py:862, in BaseBackend.reconnect(self)
    860 def reconnect(self) -> None:
    861     """Reconnect to the database already configured with connect."""
--> 862     self.do_connect(*self._con_args, **self._con_kwargs)

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:212, in Backend.do_connect(self, host, port, database, timeout, use_ssl, ca_cert, user, password, auth_mechanism, kerberos_service_name, pool_size, **params)
    210 self.con = con
    211 self.options = {}
--> 212 self._ensure_temp_db_exists()

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:668, in Backend._ensure_temp_db_exists(self)
    666 name, path = options.impala.temp_db, options.impala.temp_path
    667 if name not in self.list_databases():
--> 668     self.create_database(name, path=path, force=True)

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:307, in Backend.create_database(self, name, path, force)
    295 """Create a new Impala database.
    296 
    297 Parameters
   (...)
    304     Forcibly create the database
    305 """
    306 statement = CreateDatabase(name, path=path, can_exist=force)
--> 307 self._safe_exec_sql(statement)

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:276, in Backend._safe_exec_sql(self, *args, **kwargs)
    275 def _safe_exec_sql(self, *args, **kwargs):
--> 276     with self._safe_raw_sql(*args, **kwargs):
    277         pass

File c:\Mambaforge\envs\env_name\Lib\contextlib.py:137, in _GeneratorContextManager.__enter__(self)
    135 del self.args, self.kwds, self.func
    136 try:
--> 137     return next(self.gen)
    138 except StopIteration:
    139     raise RuntimeError("generator didn't yield") from None

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:272, in Backend._safe_raw_sql(self, query)
    270 if not isinstance(query, str):
    271     query = query.compile()
--> 272 with contextlib.closing(self.raw_sql(query)) as cur:
    273     yield cur

File c:\Mambaforge\envs\env_name\Lib\site-packages\ibis\backends\impala\__init__.py:252, in Backend.raw_sql(self, query)
    249     cursor._wait_to_finish()
    251     util.log(query)
--> 252     cursor.execute_async(query)
    254     cursor._wait_to_finish()
    255 except (Exception, KeyboardInterrupt):

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:388, in HiveServer2Cursor.execute_async(self, operation, parameters, configuration)
    383     op = self.session.execute(self._last_operation_string,
    384                               configuration,
    385                               run_async=True)
    386     self._last_operation = op
--> 388 self._execute_async(op)

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:407, in HiveServer2Cursor._execute_async(self, operation_fn)
    405 self._reset_state()
    406 self._debug_log_state()
--> 407 operation_fn()
    408 self._last_operation_active = True
    409 self._debug_log_state()

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:383, in HiveServer2Cursor.execute_async.<locals>.op()
    380 else:
    381     self._last_operation_string = operation
--> 383 op = self.session.execute(self._last_operation_string,
    384                           configuration,
    385                           run_async=True)
    386 self._last_operation = op

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:1227, in HS2Session.execute(self, statement, configuration, run_async)
   1219 req = TExecuteStatementReq(sessionHandle=self.handle,
   1220                            statement=statement,
   1221                            confOverlay=configuration,
   1222                            runAsync=run_async)
   1223 # Do not try to retry http requests.
   1224 # Read queries should be idempotent but most dml queries are not. Also retrying
   1225 # query execution from client could be expensive and so likely makes sense to do
   1226 # it if server is also aware of the retries.
-> 1227 return self._operation('ExecuteStatement', req, False)

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:1148, in ThriftRPC._operation(self, kind, request, retry_on_http_error)
   1147 def _operation(self, kind, request, retry_on_http_error=False):
-> 1148     resp = self._rpc(kind, request, retry_on_http_error)
   1149     return self._get_operation(resp.operationHandle)

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:1085, in ThriftRPC._rpc(self, func_name, request, retry_on_http_error)
   1083 response = self._execute(func_name, request, retry_on_http_error)
   1084 self._log_response(func_name, response)
-> 1085 err_if_rpc_not_ok(response)
   1086 return response

File c:\Mambaforge\envs\env_name\Lib\site-packages\impala\hiveserver2.py:781, in err_if_rpc_not_ok(resp)
    777 def err_if_rpc_not_ok(resp):
    778     if (resp.status.statusCode != TStatusCode.SUCCESS_STATUS and
    779             resp.status.statusCode != TStatusCode.SUCCESS_WITH_INFO_STATUS and
    780             resp.status.statusCode != TStatusCode.STILL_EXECUTING_STATUS):
--> 781         raise HiveServer2Error(resp.status.errorMessage)

HiveServer2Error: AuthorizationException: User 'username' does not have privileges to execute 'CREATE' on: __ibis_tmp
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Thanks for reporting (again)!

A quick glance at the code suggests the temporary database may not be needed at all. I'll poke at it a bit.

Resolving patch:
diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py
--- a/ibis/backends/impala/__init__.py
+++ b/ibis/backends/impala/__init__.py
@@ -41,7 +41,6 @@
     wrap_udf,
 )
 from ibis.backends.sql import SQLBackend
-from ibis.config import options
 
 if TYPE_CHECKING:
     from collections.abc import Iterator, Mapping
@@ -68,21 +67,6 @@ class Backend(SQLBackend):
 
     supports_in_memory_tables = True
 
-    class Options(ibis.config.Config):
-        """Impala specific options.
-
-        Parameters
-        ----------
-        temp_db : str, default "__ibis_tmp"
-            Database to use for temporary objects.
-        temp_path : str, default "/tmp/ibis"
-            Path for storage of temporary data.
-
-        """
-
-        temp_db: str = "__ibis_tmp"
-        temp_path: str = "/tmp/__ibis"
-
     def _from_url(self, url: str, **kwargs: Any) -> Backend:
         """Connect to a backend using a URL `url`.
 
@@ -212,7 +196,6 @@ def do_connect(
 
         self.con = con
         self.options = {}
-        self._ensure_temp_db_exists()
 
     @cached_property
     def version(self):
@@ -703,12 +686,6 @@ def _get_concrete_table_path(
     ) -> tuple[str, str | None]:
         return name if name is not None else util.gen_name("impala_table"), database
 
-    def _ensure_temp_db_exists(self):
-        # TODO: session memoize to avoid unnecessary `SHOW DATABASES` calls
-        name, path = options.impala.temp_db, options.impala.temp_path
-        if name not in self.list_databases():
-            self.create_database(name, path=path, force=True)
-
     def _drop_table(self, name: str) -> None:
         # database might have been dropped, so we suppress the
         # corresponding Exception


Test patch:
diff --git a/ibis/backends/impala/tests/conftest.py b/ibis/backends/impala/tests/conftest.py
--- a/ibis/backends/impala/tests/conftest.py
+++ b/ibis/backends/impala/tests/conftest.py
@@ -154,10 +154,7 @@ def impala_port(self):
     @property
     def tmp_dir(self):
         leaf = util.gen_name("impala_test_tmp_dir")
-        options.impala.temp_path = tmp_dir = os.environ.get(
-            "IBIS_TEST_TMP_DIR", f"/tmp/{leaf}"
-        )
-        return tmp_dir
+        return os.environ.get("IBIS_TEST_TMP_DIR", f"/tmp/{leaf}")
 
     @property
     def test_data_db(self):
@@ -175,8 +172,7 @@ def env():
 
 @pytest.fixture(scope="session")
 def tmp_dir(env):
-    options.impala.temp_path = tmp_dir = env.tmp_dir
-    return tmp_dir
+    return env.tmp_dir
 
 
 @pytest.fixture(scope="session")