Repository: ibis-project/ibis

Issue:
test: pandas and dask backends tests are not inferring the correct schema
The next tests were xfailed because pandas and dask don't implement `drop_table`, even if it seems somehow unrelated:

- `ibis/backends/tests/test_client.py::test_create_table_from_schema`
- `ibis/backends/tests/test_client.py::test_nullable_input_output`

When executed, the tests fail because nullable fields in the schema are not nullable in the created table.

We moved to xfail the tests only when `create_table` is not implemented, ignoring `drop_table`, which is not used in the test. Manually xfailing the tests for pandas and dask in #3018, but we probably want to revisit this, and see if it's a bug.


Discussion:
The reason these tests are failing is unrelated to the presence of nulls.

Again, xfailing tests without understanding why they are failing leads to a bunch of work that has to be done later.

Resolving patch:
diff --git a/ibis/backends/dask/__init__.py b/ibis/backends/dask/__init__.py
--- a/ibis/backends/dask/__init__.py
+++ b/ibis/backends/dask/__init__.py
@@ -1,15 +1,15 @@
-from typing import Mapping
+from __future__ import annotations
+
+from typing import Any, Mapping
 
 import dask
 import dask.dataframe as dd
 import pandas as pd
-import toolz
 from dask.base import DaskMethodsMixin
 
 # import the pandas execution module to register dispatched implementations of
 # execute_node that the dask backend will later override
 import ibis.backends.pandas.execution  # noqa: F401
-import ibis.common.exceptions as com
 import ibis.config
 import ibis.expr.schema as sch
 import ibis.expr.types as ir
@@ -26,6 +26,7 @@ class Backend(BasePandasBackend):
     name = 'dask'
     database_class = DaskDatabase
     table_class = DaskTable
+    backend_table_type = dd.DataFrame
 
     def do_connect(self, dictionary):
         # register dispatchers
@@ -72,28 +73,30 @@ def compile(
         -----
         For the dask backend returns a dask graph that you can run ``.compute``
         on to get a pandas object.
-
         """
         return execute_and_reset(query, params=params, **kwargs)
 
+    @classmethod
+    def _supports_conversion(cls, obj: Any) -> bool:
+        return isinstance(obj, cls.backend_table_type)
+
+    @staticmethod
+    def _from_pandas(df: pd.DataFrame, npartitions: int = 1) -> dd.DataFrame:
+        return dd.from_pandas(df, npartitions=npartitions)
+
+    @staticmethod
+    def _convert_schema(schema: sch.Schema):
+        return ibis_schema_to_dask(schema)
+
+    @classmethod
+    def _convert_object(cls, obj: dd.DataFrame) -> dd.DataFrame:
+        return obj
+
     def create_table(
         self,
         table_name: str,
-        obj: dd.DataFrame = None,
-        schema: sch.Schema = None,
+        obj: dd.DataFrame | None = None,
+        schema: sch.Schema | None = None,
     ):
         """Create a table."""
-        if obj is not None:
-            df = obj
-        elif schema is not None:
-            dtypes = ibis_schema_to_dask(schema)
-            df = schema.apply_to(
-                dd.from_pandas(
-                    pd.DataFrame(columns=list(map(toolz.first, dtypes))),
-                    npartitions=1,
-                )
-            )
-        else:
-            raise com.IbisError('Must pass expr or schema')
-
-        self.dictionary[table_name] = df
+        super().create_table(table_name, obj=obj, schema=schema)
diff --git a/ibis/backends/pandas/__init__.py b/ibis/backends/pandas/__init__.py
--- a/ibis/backends/pandas/__init__.py
+++ b/ibis/backends/pandas/__init__.py
@@ -1,5 +1,8 @@
+from __future__ import annotations
+
+from typing import Any, MutableMapping
+
 import pandas as pd
-import toolz
 
 import ibis.common.exceptions as com
 import ibis.config
@@ -15,34 +18,37 @@ class BasePandasBackend(BaseBackend):
     Base class for backends based on pandas.
     """
 
-    def do_connect(self, dictionary):
-        """Construct a client from a dictionary of DataFrames.
+    backend_table_type = pd.DataFrame
+
+    def do_connect(
+        self,
+        dictionary: MutableMapping[str, pd.DataFrame],
+    ) -> None:
+        """Construct a client from a dictionary of pandas DataFrames.
 
         Parameters
         ----------
-        dictionary : dict
-
-        Returns
-        -------
-        Backend
+        dictionary
+            Dictionary mapping string table names to pandas DataFrames.
         """
         # register dispatchers
         from . import execution  # noqa F401
         from . import udf  # noqa F401
 
         self.dictionary = dictionary
+        self.schemas: MutableMapping[str, sch.Schema] = {}
 
     def from_dataframe(self, df, name='df', client=None):
-        """
-        convenience function to construct an ibis table
-        from a DataFrame
+        """Construct an ibis table from a pandas DataFrame.
 
         Parameters
         ----------
-        df : DataFrame
-        name : str, default 'df'
-        client : Backend, optional
-            client dictionary will be mutated with the name of the DataFrame,
+        df
+            A pandas DataFrame
+        name
+            The name of the pandas DataFrame
+        client
+            Client dictionary will be mutated with the name of the DataFrame,
             if not provided a new client is created
 
         Returns
@@ -79,7 +85,7 @@ def list_tables(self, like=None, database=None):
 
     def table(self, name: str, schema: sch.Schema = None):
         df = self.dictionary[name]
-        schema = sch.infer(df, schema=schema)
+        schema = sch.infer(df, schema=schema or self.schemas.get(name, None))
         return self.table_class(name, schema, self).to_expr()
 
     def database(self, name=None):
@@ -90,11 +96,58 @@ def load_data(self, table_name, obj, **kwargs):
         self.dictionary[table_name] = obj
 
     def get_schema(self, table_name, database=None):
-        return sch.infer(self.dictionary[table_name])
+        schemas = self.schemas
+        try:
+            schema = schemas[table_name]
+        except KeyError:
+            schemas[table_name] = schema = sch.infer(
+                self.dictionary[table_name]
+            )
+        return schema
 
     def compile(self, expr, *args, **kwargs):
         return expr
 
+    def create_table(self, table_name, obj=None, schema=None):
+        """Create a table."""
+        if obj is None and schema is None:
+            raise com.IbisError('Must pass expr or schema')
+
+        if obj is not None:
+            if not self._supports_conversion(obj):
+                raise com.BackendConversionError(
+                    f"Unable to convert {obj.__class__} object "
+                    f"to backend type: {self.__class__.backend_table_type}"
+                )
+            df = self._convert_object(obj)
+        else:
+            pandas_schema = self._convert_schema(schema)
+            dtypes = dict(pandas_schema)
+            df = self._from_pandas(
+                pd.DataFrame(columns=dtypes.keys()).astype(dtypes)
+            )
+
+        self.dictionary[table_name] = df
+
+        if schema is not None:
+            self.schemas[table_name] = schema
+
+    @classmethod
+    def _supports_conversion(cls, obj: Any) -> bool:
+        return True
+
+    @staticmethod
+    def _convert_schema(schema: sch.Schema):
+        return ibis_schema_to_pandas(schema)
+
+    @staticmethod
+    def _from_pandas(df: pd.DataFrame) -> pd.DataFrame:
+        return df
+
+    @classmethod
+    def _convert_object(cls, obj: Any) -> Any:
+        return cls.backend_table_type(obj)
+
 
 class Backend(BasePandasBackend):
     name = 'pandas'
@@ -117,18 +170,3 @@ def execute(self, query, params=None, limit='default', **kwargs):
                 )
             )
         return execute_and_reset(query, params=params, **kwargs)
-
-    def create_table(self, table_name, obj=None, schema=None):
-        """Create a table."""
-        if obj is None and schema is None:
-            raise com.IbisError('Must pass expr or schema')
-
-        if obj is not None:
-            df = pd.DataFrame(obj)
-        else:
-            dtypes = ibis_schema_to_pandas(schema)
-            df = schema.apply_to(
-                pd.DataFrame(columns=list(map(toolz.first, dtypes)))
-            )
-
-        self.dictionary[table_name] = df
diff --git a/ibis/common/exceptions.py b/ibis/common/exceptions.py
--- a/ibis/common/exceptions.py
+++ b/ibis/common/exceptions.py
@@ -71,6 +71,10 @@ class UnsupportedArgumentError(IbisError):
     """UnsupportedArgumentError."""
 
 
+class BackendConversionError(IbisError):
+    """A backend cannot convert an input to its native type."""
+
+
 def mark_as_unsupported(f: Callable) -> Callable:
     """Decorate an unsupported method.
 


Test patch:
diff --git a/ibis/backends/dask/tests/test_client.py b/ibis/backends/dask/tests/test_client.py
--- a/ibis/backends/dask/tests/test_client.py
+++ b/ibis/backends/dask/tests/test_client.py
@@ -52,7 +52,8 @@ def test_load_data(client, npartitions):
 
 
 def test_create_table(client, npartitions):
-    client.create_table('testing', obj=make_dask_data_frame(npartitions))
+    ddf = make_dask_data_frame(npartitions)
+    client.create_table('testing', obj=ddf)
     assert 'testing' in client.list_tables()
     client.create_table('testingschema', schema=client.get_schema('testing'))
     assert 'testingschema' in client.list_tables()
diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py
--- a/ibis/backends/tests/test_client.py
+++ b/ibis/backends/tests/test_client.py
@@ -96,13 +96,8 @@ def test_sql(backend, con, sql):
     con.sql(sql).execute()
 
 
-# test table
-
-
 @pytest.mark.xfail_unsupported
-@pytest.mark.xfail_backends(['pandas', 'dask'])
 def test_create_table_from_schema(con, backend, new_schema, temp_table):
-    # xfailing pandas and dask: #3020
     con.create_table(temp_table, schema=new_schema)
 
     t = con.table(temp_table)
@@ -114,7 +109,7 @@ def test_create_table_from_schema(con, backend, new_schema, temp_table):
 @pytest.mark.xfail_unsupported
 def test_rename_table(con, backend, temp_table, new_schema):
     if not hasattr(con, 'rename_table'):
-        pytest.xfail('{} backend doesn\'t have rename_table method.')
+        pytest.xfail(f"{backend.name} backend has no `rename_table` method")
 
     temp_table_original = f'{temp_table}_original'
     con.create_table(temp_table_original, schema=new_schema)
@@ -127,10 +122,9 @@ def test_rename_table(con, backend, temp_table, new_schema):
 
 
 @pytest.mark.xfail_unsupported
-@pytest.mark.xfail_backends(['impala', 'pyspark', 'spark', 'pandas', 'dask'])
+@pytest.mark.xfail_backends(['impala', 'pyspark', 'spark'])
 def test_nullable_input_output(con, backend, temp_table):
     # - Impala, PySpark and Spark non-nullable issues #2138 and #2137
-    # xfailing pandas and dask: #3020
     sch = ibis.schema(
         [
             ('foo', 'int64'),