Repository: ibis-project/ibis

Issue:
feat: add `catalog` & `database` support to `create_table`
### What happened?

## TLDR
I'm wondering if it's intended that the `database` argument in `create_table` works different than the one in `drop_table`? `create_table` only accepts a `str` and `drop_table` accepts a `tuple`. 

If I set the catalog and database via pyspark, `create_table` works as excepted, but I can't figure out a way to do so in my `create_table`, I had to do it through the pyspark session directly:
```
from pyspark.sql import SparkSession
import ibis
spark = SparkSession.builder.getOrCreate()
ispark = ibis.pyspark.connect(session = spark)
ispark._session.catalog.setCurrentCatalog("comms_media_dev")
ispark._session.catalog.setCurrentDatabase("dart_extensions")
ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta")
```
I can drop a table without accessing the pyspark session:
```
ispark.drop_table(name = "raw_camp_info", database=tuple(["comms_media_dev", "dart_extensions"]))
```

## Additional Details
To drop my table I can just specify the catalog and database in my call:
```
from pyspark.sql import SparkSession
import ibis
spark = SparkSession.builder.getOrCreate()
ispark = ibis.pyspark.connect(session = spark)
ispark.drop_table(name = "raw_camp_info", database=tuple(["comms_media_dev", "dart_extensions"]))
```

Trying the same approach with `create_table` if fails:
```
ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta", database=tuple(["comms_media_dev", "dart_extensions"]))
```
> py4j.Py4JException: Method setCurrentDatabase([class java.util.ArrayList]) does not exist

I also tried with dot separator:
```
ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta", database="comms_media_dev.dart_extensions")
```
> AnalysisException: too many name parts for schema name: comms_media_dev.dart_extensions

I then tried to set the catalog and provide the database name and got a permissions error. Looking through the error, it looks like create_table didn't pass the `database` argument because the database was set to default (i.e. `comms_media_dev.default`:
```
ispark._session.catalog.setCurrentCatalog("comms_media_dev")
ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta", database="dart_extensions")
```

> com.databricks.sql.managedcatalog.acl.UnauthorizedAccessException: PERMISSION_DENIED: User does not have USE SCHEMA on Schema 'comms_media_dev.default'.
> ---------------------------------------------------------------------------
> Py4JJavaError                             Traceback (most recent call last)
> File <command-1124948972684715>:10
>       8 df = ispark.read_parquet(source = "abfss://my_parquet")
>       9 df = df.select(_.CAMP_START_DATE, _.CAMP_END_DATE, _.KPM_PROJECT_ID)
> ---> 10 ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta", database="dart_extensions")
> 
> File /local_disk0/.ephemeral_nfs/envs/pythonEnv-c1f2e765-cb45-4a96-a1dc-bd25ecd4f460/lib/python3.9/site-packages/ibis/backends/pyspark/__init__.py:453, in Backend.create_table(self, name, obj, schema, database, temp, overwrite, format)
>     451         self._run_pre_execute_hooks(table)
>     452         df = self._session.sql(query)
> --> 453         df.write.saveAsTable(name, format=format, mode=mode)
>     454 elif schema is not None:
>     455     schema = PySparkSchema.from_ibis(schema)
> 
> File /usr/lib/python3.9/contextlib.py:124, in _GeneratorContextManager.__exit__(self, type, value, traceback)
>     122 if type is None:
>     123     try:
> --> 124         next(self.gen)
>     125     except StopIteration:
>     126         return False
> 
> File /local_disk0/.ephemeral_nfs/envs/pythonEnv-c1f2e765-cb45-4a96-a1dc-bd25ecd4f460/lib/python3.9/site-packages/ibis/backends/pyspark/__init__.py:234, in Backend._active_database(self, name)
>     232     yield
>     233 finally:
> --> 234     self._session.catalog.setCurrentDatabase(current)
> 
> File /databricks/spark/python/pyspark/instrumentation_utils.py:48, in _wrap_function.<locals>.wrapper(*args, **kwargs)
>      46 start = time.perf_counter()
>      47 try:
> ---> 48     res = func(*args, **kwargs)
>      49     logger.log_success(
>      50         module_name, class_name, function_name, time.perf_counter() - start, signature
>      51     )
>      52     return res
> 
> File /databricks/spark/python/pyspark/sql/catalog.py:185, in Catalog.setCurrentDatabase(self, dbName)
>     172 def setCurrentDatabase(self, dbName: str) -> None:
>     173     """
>     174     Sets the current default database in this session.
>     175 
>    (...)
>     183     >>> spark.catalog.setCurrentDatabase("default")
>     184     """
> --> 185     return self._jcatalog.setCurrentDatabase(dbName)
> 
> File /databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py:1321, in JavaMember.__call__(self, *args)
>    1315 command = proto.CALL_COMMAND_NAME +\
>    1316     self.command_header +\
>    1317     args_command +\
>    1318     proto.END_COMMAND_PART
>    1320 answer = self.gateway_client.send_command(command)
> -> 1321 return_value = get_return_value(
>    1322     answer, self.gateway_client, self.target_id, self.name)
>    1324 for temp_arg in temp_args:
>    1325     temp_arg._detach()
> 
> File /databricks/spark/python/pyspark/errors/exceptions.py:228, in capture_sql_exception.<locals>.deco(*a, **kw)
>     226 def deco(*a: Any, **kw: Any) -> Any:
>     227     try:
> --> 228         return f(*a, **kw)
>     229     except Py4JJavaError as e:
>     230         converted = convert_exception(e.java_exception)
> 
> File /databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
>     324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
>     325 if answer[1] == REFERENCE_TYPE:
> --> 326     raise Py4JJavaError(
>     327         "An error occurred while calling {0}{1}{2}.\n".
>     328         format(target_id, ".", name), value)
>     329 else:
>     330     raise Py4JError(
>     331         "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".
>     332         format(target_id, ".", name, value))
> 
> Py4JJavaError: An error occurred while calling o569.setCurrentDatabase.
> : com.databricks.sql.managedcatalog.acl.UnauthorizedAccessException: PERMISSION_DENIED: User does not have USE SCHEMA on Schema 'comms_media_dev.default'.
> 	at com.databricks.managedcatalog.UCReliableHttpClient.reliablyAndTranslateExceptions(UCReliableHttpClient.scala:84)
> 	at com.databricks.managedcatalog.UCReliableHttpClient.get(UCReliableHttpClient.scala:136)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.$anonfun$getSchema$1(ManagedCatalogClientImpl.scala:378)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.$anonfun$recordAndWrapException$2(ManagedCatalogClientImpl.scala:3401)
> 	at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:80)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.$anonfun$recordAndWrapException$1(ManagedCatalogClientImpl.scala:3400)
> 	at com.databricks.managedcatalog.ErrorDetailsHandler.wrapServiceException(ErrorDetailsHandler.scala:25)
> 	at com.databricks.managedcatalog.ErrorDetailsHandler.wrapServiceException$(ErrorDetailsHandler.scala:23)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.wrapServiceException(ManagedCatalogClientImpl.scala:106)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.recordAndWrapException(ManagedCatalogClientImpl.scala:3397)
> 	at com.databricks.managedcatalog.ManagedCatalogClientImpl.getSchema(ManagedCatalogClientImpl.scala:374)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogCommon.$anonfun$getSchemaMetadata$5(ManagedCatalogCommon.scala:227)
> 	at scala.Option.getOrElse(Option.scala:189)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogCommon.getSchemaMetadata(ManagedCatalogCommon.scala:227)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogCommon.schemaExists(ManagedCatalogCommon.scala:233)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogSessionCatalog.databaseExists(ManagedCatalogSessionCatalog.scala:631)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogSessionCatalog.requireScExists(ManagedCatalogSessionCatalog.scala:274)
> 	at com.databricks.sql.managedcatalog.ManagedCatalogSessionCatalog.setCurrentDatabase(ManagedCatalogSessionCatalog.scala:492)
> 	at com.databricks.sql.DatabricksCatalogManager.setCurrentNamespace(DatabricksCatalogManager.scala:159)
> 	at org.apache.spark.sql.internal.CatalogImpl.setCurrentDatabase(CatalogImpl.scala:98)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
> 	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
> 	at py4j.Gateway.invoke(Gateway.java:306)
> 	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
> 	at py4j.commands.CallCommand.execute(CallCommand.java:79)
> 	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:195)
> 	at py4j.ClientServerConnection.run(ClientServerConnection.java:115)
> 	at java.lang.Thread.run(Thread.java:750)

If I set the catalog and database via pyspark, `create_table` works as excepted:
```
ispark._session.catalog.setCurrentCatalog("comms_media_dev")
ispark._session.catalog.setCurrentDatabase("dart_extensions")
ispark.create_table(name = "raw_camp_info", obj = df, overwrite = True, format="delta")
```
### What version of ibis are you using?

https://github.com/ibis-project/ibis.git@93552812ee9e8e0e3397bc226cc20c381fcd173b 

### What backend(s) are you using, if any?

pyspark

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:

Resolving patch:
diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py
--- a/ibis/backends/pyspark/__init__.py
+++ b/ibis/backends/pyspark/__init__.py
@@ -8,6 +8,7 @@
 import pyspark
 import sqlglot as sg
 import sqlglot.expressions as sge
+from packaging.version import parse as vparse
 from pyspark import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import PandasUDFType, pandas_udf
@@ -19,7 +20,7 @@
 import ibis.expr.schema as sch
 import ibis.expr.types as ir
 from ibis import util
-from ibis.backends import CanCreateDatabase
+from ibis.backends import CanCreateDatabase, CanListCatalog
 from ibis.backends.pyspark.compiler import PySparkCompiler
 from ibis.backends.pyspark.converter import PySparkPandasData
 from ibis.backends.pyspark.datatypes import PySparkSchema, PySparkType
@@ -33,6 +34,8 @@
     import pandas as pd
     import pyarrow as pa
 
+PYSPARK_LT_34 = vparse(pyspark.__version__) < vparse("3.4")
+
 
 def normalize_filenames(source_list):
     # Promote to list
@@ -127,7 +130,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         """No-op for compatibility."""
 
 
-class Backend(SQLBackend, CanCreateDatabase):
+class Backend(SQLBackend, CanListCatalog, CanCreateDatabase):
     name = "pyspark"
     compiler = PySparkCompiler()
 
@@ -221,6 +224,11 @@ def current_database(self) -> str:
         [(db,)] = self._session.sql("SELECT CURRENT_DATABASE()").collect()
         return db
 
+    @property
+    def current_catalog(self) -> str:
+        [(catalog,)] = self._session.sql("SELECT CURRENT_CATALOG()").collect()
+        return catalog
+
     @contextlib.contextmanager
     def _active_database(self, name: str | None):
         if name is None:
@@ -233,10 +241,29 @@ def _active_database(self, name: str | None):
         finally:
             self._session.catalog.setCurrentDatabase(current)
 
-    def list_databases(self, like: str | None = None) -> list[str]:
-        databases = [
-            db.namespace for db in self._session.sql("SHOW DATABASES").collect()
-        ]
+    @contextlib.contextmanager
+    def _active_catalog(self, name: str | None):
+        if name is None or PYSPARK_LT_34:
+            yield
+            return
+        current = self.current_catalog
+        try:
+            self._session.catalog.setCurrentCatalog(name)
+            yield
+        finally:
+            self._session.catalog.setCurrentCatalog(current)
+
+    def list_catalogs(self, like: str | None = None) -> list[str]:
+        catalogs = [res.catalog for res in self._session.sql("SHOW CATALOGS").collect()]
+        return self._filter_with_like(catalogs, like)
+
+    def list_databases(
+        self, like: str | None = None, catalog: str | None = None
+    ) -> list[str]:
+        with self._active_catalog(catalog):
+            databases = [
+                db.namespace for db in self._session.sql("SHOW DATABASES").collect()
+            ]
         return self._filter_with_like(databases, like)
 
     def list_tables(
@@ -250,14 +277,21 @@ def list_tables(
             A pattern to use for listing tables.
         database
             Database to list tables from. Default behavior is to show tables in
-            the current database.
+            the current catalog and database.
+
+            To specify a table in a separate catalog, you can pass in the
+            catalog and database as a string `"catalog.database"`, or as a tuple of
+            strings `("catalog", "database")`.
         """
-        tables = [
-            row.tableName
-            for row in self._session.sql(
-                f"SHOW TABLES IN {database or self.current_database}"
-            ).collect()
-        ]
+        table_loc = self._to_sqlglot_table(database)
+        catalog, db = self._to_catalog_db_tuple(table_loc)
+        with self._active_catalog(catalog):
+            tables = [
+                row.tableName
+                for row in self._session.sql(
+                    f"SHOW TABLES IN {db or self.current_database}"
+                ).collect()
+            ]
         return self._filter_with_like(tables, like)
 
     def _wrap_udf_to_return_pandas(self, func, output_dtype):
@@ -319,6 +353,8 @@ def raw_sql(self, query: str | sg.Expression, **kwargs: Any) -> _PySparkCursor:
     def create_database(
         self,
         name: str,
+        *,
+        catalog: str | None = None,
         path: str | Path | None = None,
         force: bool = False,
     ) -> Any:
@@ -328,6 +364,8 @@ def create_database(
         ----------
         name
             Database name
+        catalog
+            Catalog to create database in (defaults to ``current_catalog``)
         path
             Path where to store the database data; otherwise uses Spark default
         force
@@ -347,16 +385,21 @@ def create_database(
             this=sg.to_identifier(name),
             properties=properties,
         )
-        with self._safe_raw_sql(sql):
-            pass
+        with self._active_catalog(catalog):
+            with self._safe_raw_sql(sql):
+                pass
 
-    def drop_database(self, name: str, force: bool = False) -> Any:
+    def drop_database(
+        self, name: str, *, catalog: str | None = None, force: bool = False
+    ) -> Any:
         """Drop a Spark database.
 
         Parameters
         ----------
         name
             Database name
+        catalog
+            Catalog containing database to drop (defaults to ``current_catalog``)
         force
             If False, Spark throws exception if database is not empty or
             database does not exist
@@ -365,8 +408,9 @@ def drop_database(self, name: str, force: bool = False) -> Any:
         sql = sge.Drop(
             kind="DATABASE", exist=force, this=sg.to_identifier(name), cascade=force
         )
-        with self._safe_raw_sql(sql):
-            pass
+        with self._active_catalog(catalog):
+            with self._safe_raw_sql(sql):
+                pass
 
     def get_schema(
         self,
@@ -382,7 +426,7 @@ def get_schema(
         table_name
             Table name. May be fully qualified
         catalog
-            Unsupported in PySpark backend.
+            Catalog to use
         database
             Database to use to get the active database.
 
@@ -392,7 +436,10 @@ def get_schema(
             An ibis schema
 
         """
-        with self._active_database(database):
+
+        table_loc = self._to_sqlglot_table((catalog, database))
+        catalog, db = self._to_catalog_db_tuple(table_loc)
+        with self._active_catalog(catalog), self._active_database(db):
             df = self._session.table(table_name)
             struct = PySparkType.to_ibis(df.schema)
 
@@ -421,8 +468,12 @@ def create_table(
             Mutually exclusive with `obj`, creates an empty table with a schema
         database
             Database name
+
+            To specify a table in a separate catalog, you can pass in the
+            catalog and database as a string `"catalog.database"`, or as a tuple of
+            strings `("catalog", "database")`.
         temp
-            Whether the new table is temporary
+            Whether the new table is temporary (unsupported)
         overwrite
             If `True`, overwrite existing data
         format
@@ -443,22 +494,25 @@ def create_table(
                 "PySpark backend does not yet support temporary tables"
             )
 
+        table_loc = self._to_sqlglot_table(database)
+        catalog, db = self._to_catalog_db_tuple(table_loc)
+
         if obj is not None:
             table = obj if isinstance(obj, ir.Expr) else ibis.memtable(obj)
             query = self.compile(table)
             mode = "overwrite" if overwrite else "error"
-            with self._active_database(database):
+            with self._active_catalog(catalog), self._active_database(db):
                 self._run_pre_execute_hooks(table)
                 df = self._session.sql(query)
                 df.write.saveAsTable(name, format=format, mode=mode)
         elif schema is not None:
             schema = PySparkSchema.from_ibis(schema)
-            with self._active_database(database):
+            with self._active_catalog(catalog), self._active_database(db):
                 self._session.catalog.createTable(name, schema=schema, format=format)
         else:
             raise com.IbisError("The schema or obj parameter is required")
 
-        return self.table(name, database=database)
+        return self.table(name, database=db)
 
     def create_view(
         self,


Test patch:
diff --git a/ibis/backends/pyspark/tests/test_client.py b/ibis/backends/pyspark/tests/test_client.py
new file mode 100644
--- /dev/null
+++ b/ibis/backends/pyspark/tests/test_client.py
@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import ibis
+
+
+def test_catalog_db_args(con, monkeypatch):
+    monkeypatch.setattr(ibis.options, "default_backend", con)
+    t = ibis.memtable({"epoch": [1712848119, 1712848121, 1712848155]})
+
+    # create a table in specified catalog and db
+    con.create_table(
+        "t2", database=(con.current_catalog, "default"), obj=t, overwrite=True
+    )
+
+    assert "t2" not in con.list_tables()
+    assert "t2" in con.list_tables(database="default")
+    assert "t2" in con.list_tables(database="spark_catalog.default")
+    assert "t2" in con.list_tables(database=("spark_catalog", "default"))
+
+    con.drop_table("t2", database="spark_catalog.default")
+
+    assert "t2" not in con.list_tables(database="default")
diff --git a/ibis/backends/tests/test_api.py b/ibis/backends/tests/test_api.py
--- a/ibis/backends/tests/test_api.py
+++ b/ibis/backends/tests/test_api.py
@@ -32,7 +32,6 @@ def test_version(backend):
         "bigquery",
         "mysql",
         "impala",
-        "pyspark",
         "flink",
     ],
     reason="backend does not support catalogs",
@@ -43,6 +42,7 @@ def test_version(backend):
     raises=NotImplementedError,
     reason="current_catalog isn't implemented",
 )
+@pytest.mark.xfail_version(pyspark=["pyspark<3.4"])
 def test_catalog_consistency(backend, con):
     catalogs = con.list_catalogs()
     assert isinstance(catalogs, list)
diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py
--- a/ibis/backends/tests/test_client.py
+++ b/ibis/backends/tests/test_client.py
@@ -615,12 +615,12 @@ def test_insert_from_memtable(con, temp_table):
         "pandas",
         "polars",
         "flink",
-        "pyspark",
         "sqlite",
     ],
     raises=AttributeError,
     reason="doesn't support the common notion of a catalog",
 )
+@pytest.mark.xfail_version(pyspark=["pyspark<3.4"])
 def test_list_catalogs(con):
     # Every backend has its own databases
     test_catalogs = {
@@ -634,6 +634,7 @@ def test_list_catalogs(con):
         "risingwave": {"dev"},
         "snowflake": {"IBIS_TESTING"},
         "trino": {"memory"},
+        "pyspark": {"spark_catalog"},
     }
     result = set(con.list_catalogs())
     assert test_catalogs[con.name] <= result
@@ -647,7 +648,7 @@ def test_list_catalogs(con):
         "polars",
     ],
     raises=AttributeError,
-    reason="doesn't support the common notion of a catalog",
+    reason="doesn't support the common notion of a database",
 )
 def test_list_database_contents(con):
     # Every backend has its own databases