Repository: modin-project/modin

Issue:
Dataframes with categorical columns cannot be interchanged
Interchanging a dataframe with categorical columns results in `NotImplementedError`. I'm not super familiar with data storage here, but I was wondering if maybe this is actually an unintended bug.

```python
>>> from modin import pandas as pd
>>> df = pd.DataFrame({"foo": pd.Series([0, 1, 2], dtype="category")})
>>> from modin.pandas.utils import from_dataframe
>>> modin_df = from_dataframe(df)
NotImplementedError: See self.describe_null
```

<details>
<summary>Full traceback</summary>

```python
.../modin/pandas/utils.py:123, in from_dataframe(df)
    120 from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher
    121 from .dataframe import DataFrame
--> 123 return DataFrame(query_compiler=FactoryDispatcher.from_dataframe(df))

.../modin/core/execution/dispatching/factories/dispatcher.py:175, in FactoryDispatcher.from_dataframe(cls, *args, **kwargs)
    172 @classmethod
    173 @_inherit_docstrings(factories.BaseFactory._from_dataframe)
    174 def from_dataframe(cls, *args, **kwargs):
--> 175     return cls.__factory._from_dataframe(*args, **kwargs)

.../modin/core/execution/dispatching/factories/factories.py:197, in BaseFactory._from_dataframe(cls, *args, **kwargs)
    189 @classmethod
    190 @doc(
    191     _doc_io_method_template,
   (...)
    195 )
    196 def _from_dataframe(cls, *args, **kwargs):
--> 197     return cls.io_cls.from_dataframe(*args, **kwargs)

.../modin/core/io/io.py:120, in BaseIO.from_dataframe(cls, df)
    105 @classmethod
    106 def from_dataframe(cls, df):
    107     """
    108     Create a Modin QueryCompiler from a DataFrame supporting the DataFrame exchange protocol `__dataframe__()`.
    109 
   (...)
    118         QueryCompiler containing data from the DataFrame.
    119     """
--> 120     return cls.query_compiler_cls.from_dataframe(df, cls.frame_cls)

.../modin/core/storage_formats/pandas/query_compiler.py:278, in PandasQueryCompiler.from_dataframe(cls, df, data_cls)
    276 @classmethod
    277 def from_dataframe(cls, df, data_cls):
--> 278     return cls(data_cls.from_dataframe(df))

.../modin/core/dataframe/pandas/dataframe/dataframe.py:2968, in PandasDataframe.from_dataframe(cls, df)
   2963 from modin.core.dataframe.pandas.exchange.dataframe_protocol.from_dataframe import (
   2964     from_dataframe_to_pandas,
   2965 )
   2967 ErrorMessage.default_to_pandas(message="`from_dataframe`")
-> 2968 pandas_df = from_dataframe_to_pandas(df)
   2969 return cls.from_pandas(pandas_df)

.../modin/core/dataframe/pandas/exchange/dataframe_protocol/from_dataframe.py:68, in from_dataframe_to_pandas(df, n_chunks)
     66 pandas_dfs = []
     67 for chunk in df.get_chunks(n_chunks):
---> 68     pandas_df = protocol_df_chunk_to_pandas(chunk)
     69     pandas_dfs.append(pandas_df)
     71 pandas_df = pandas.concat(pandas_dfs, axis=0, ignore_index=True)

.../modin/core/dataframe/pandas/exchange/dataframe_protocol/from_dataframe.py:111, in protocol_df_chunk_to_pandas(df)
    109     columns[name], buf = primitive_column_to_ndarray(col)
    110 elif dtype == DTypeKind.CATEGORICAL:
--> 111     columns[name], buf = categorical_column_to_series(col)
    112 elif dtype == DTypeKind.STRING:
    113     columns[name], buf = string_column_to_ndarray(col)

.../modin/core/dataframe/pandas/exchange/dataframe_protocol/from_dataframe.py:167, in categorical_column_to_series(col)
    164     raise NotImplementedError("Non-dictionary categoricals not supported yet")
    166 categories = np.array(tuple(mapping.values()))
--> 167 buffers = col.get_buffers()
    169 codes_buff, codes_dtype = buffers["data"]
    170 codes = buffer_to_ndarray(codes_buff, codes_dtype, col.offset, col.size)

.../modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py:313, in PandasProtocolColumn.get_buffers(self)
    311 buffers["data"] = self._get_data_buffer()
    312 try:
--> 313     buffers["validity"] = self._get_validity_buffer()
    314 except NoValidityBuffer:
    315     buffers["validity"] = None

.../modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py:422, in PandasProtocolColumn._get_validity_buffer(self)
    420     msg = "This column uses NaN as null so does not have a separate mask"
    421 else:
--> 422     raise NotImplementedError("See self.describe_null")
    424 raise NoValidityBuffer(msg)

NotImplementedError: See self.describe_null
```

</details>

I assume the message relates to

https://github.com/modin-project/modin/blob/9b33451648a3192e93c46ac6961627ed2858c7fd/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py#L205-L207

but why can't `_get_validity_buffer()` raise `NoValidityBuffer` here instead, so by extension `get_buffers()` will have `buffers["validity"] = None`.

https://github.com/modin-project/modin/blob/9b33451648a3192e93c46ac6961627ed2858c7fd/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py#L312-L315

Wouldn't that be valid given

https://github.com/modin-project/modin/blob/9b33451648a3192e93c46ac6961627ed2858c7fd/modin/core/dataframe/base/exchange/dataframe_protocol/dataframe.py#L350-L353

i.e. if columns use sentinel values, the validity buffer can just be `None`.

cc @vnlitvinov 


Discussion:
Thanks for the report, I will have a look!

Resolving patch:
diff --git a/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py b/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py
--- a/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py
+++ b/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py
@@ -43,6 +43,13 @@
 from .exception import NoValidityBuffer, NoOffsetsBuffer
 
 
+_NO_VALIDITY_BUFFER = {
+    ColumnNullType.NON_NULLABLE: "This column is non-nullable so does not have a mask",
+    ColumnNullType.USE_NAN: "This column uses NaN as null so does not have a separate mask",
+    ColumnNullType.USE_SENTINEL: "This column uses a sentinel value so does not have a mask",
+}
+
+
 @_inherit_docstrings(ProtocolColumn)
 class PandasProtocolColumn(ProtocolColumn):
     """
@@ -414,11 +421,9 @@ def _get_validity_buffer(self) -> Tuple[PandasProtocolBuffer, Any]:
             self._validity_buffer_cache = (buffer, dtype)
             return self._validity_buffer_cache
 
-        if null == ColumnNullType.NON_NULLABLE:
-            msg = "This column is non-nullable so does not have a mask"
-        elif null == ColumnNullType.USE_NAN:
-            msg = "This column uses NaN as null so does not have a separate mask"
-        else:
+        try:
+            msg = _NO_VALIDITY_BUFFER[null]
+        except KeyError:
             raise NotImplementedError("See self.describe_null")
 
         raise NoValidityBuffer(msg)


Test patch:
diff --git a/modin/test/exchange/dataframe_protocol/pandas/test_protocol.py b/modin/test/exchange/dataframe_protocol/pandas/test_protocol.py
--- a/modin/test/exchange/dataframe_protocol/pandas/test_protocol.py
+++ b/modin/test/exchange/dataframe_protocol/pandas/test_protocol.py
@@ -19,8 +19,7 @@
 from modin.test.test_utils import warns_that_defaulting_to_pandas
 
 
-def test_simple_import():
-    modin_df_producer = pd.DataFrame(test_data["int_data"])
+def eval_df_protocol(modin_df_producer):
     internal_modin_df_producer = modin_df_producer.__dataframe__()
     # Our configuration in pytest.ini requires that we explicitly catch all
     # instances of defaulting to pandas, this one raises a warning on `.from_dataframe`
@@ -43,3 +42,13 @@ def test_simple_import():
 
     df_equals(modin_df_producer, modin_df_consumer)
     df_equals(modin_df_producer, internal_modin_df_consumer)
+
+
+def test_simple_import():
+    modin_df = pd.DataFrame(test_data["int_data"])
+    eval_df_protocol(modin_df)
+
+
+def test_categorical_from_dataframe():
+    modin_df = pd.DataFrame({"foo": pd.Series([0, 1, 2, 3], dtype="category")})
+    eval_df_protocol(modin_df)