Repository: unionai-oss/pandera

Issue:
failure_case conversion failed : polars.exceptions.ComputeError - pandera(0.19.0b3) with polars
**Describe the bug**
We are trying a simple validation example using polars. We cant understand the problem or why it originates. But it throws **polars.exceptions.ComputeError** exception when any of the validation fails and there is null in data.

For example, in the code below, the dummy data contains **extract_date** feature with a None. It runs fine if the **case_id** are all int convertible string but throws the exception if any of the case_id is not int convertible.

 
**Here is the code:**

```
import pandera.polars as pa
import polars as pl
from datetime import date
import json

class CaseSchema(pa.DataFrameModel):
    case_id: int = pa.Field(nullable=False, unique=True, coerce=True)
    gdwh_portfolio_id: str = pa.Field(nullable=False, unique=True, coerce=True)
    extract_date: date = pa.Field(nullable=True, coerce=True)

    class Config:
        drop_invalid_rows = True

invalid_lf = pl.DataFrame({
    #"case_id": ["1", "2", "3"],
    "case_id": ["1", "2", "abc"],
    "gdwh_portfolio_id": ["d", "e", "f"],
    "extract_date": [date(2024,1,1), date(2024,1,2), None]
})

try:
    CaseSchema.validate(invalid_lf, lazy=True)
except pa.errors.SchemaErrors as e:
        print(json.dumps(e.message, indent=4))

```

It gives: **'conversion from `struct[29]` to `str` failed in column 'failure_case' for 1 out of 1 values [{"abc","f",null}]**
If you uncomment  "`case_id": ["1", "2", "3"]`, and comment `"case_id": ["1", "2", "abc"]` it runs fine.

Not sure why it panics when there are nulls. If there are no nulls in the data it works fine.

The trace we get is:

```

> Traceback (most recent call last):
>   File "<frozen runpy>", line 198, in _run_module_as_main
>   File "<frozen runpy>", line 88, in _run_code
>   File "/mnt/batch/tasks/shared/LS_root/mounts/clusters/erehoba-acc-payments-req/code/Users/ourrehman/dna-payments-and-accounts/data_validation/test.py", line 22, in <module>
>     CaseSchema.validate(invalid_lf, lazy=True)
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/api/dataframe/model.py", line 289, in validate
>     cls.to_schema().validate(
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/api/polars/container.py", line 58, in validate
>     output = self.get_backend(check_obj).validate(
>              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/backends/polars/container.py", line 65, in validate
>     check_obj = parser(check_obj, *args)
>                 ^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/backends/polars/container.py", line 398, in coerce_dtype
>     check_obj = self._coerce_dtype_helper(check_obj, schema)
>                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/backends/polars/container.py", line 486, in _coerce_dtype_helper
>     raise SchemaErrors(
>           ^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/errors.py", line 183, in __init__
>     ).failure_cases_metadata(schema.name, schema_errors)
>       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/pandera/backends/polars/base.py", line 173, in failure_cases_metadata
>     ).cast(
>       ^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/polars/dataframe/frame.py", line 6624, in cast
>     return self.lazy().cast(dtypes, strict=strict).collect(_eager=True)
>            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>   File "/anaconda/envs/pandera-polars/lib/python3.11/site-packages/polars/lazyframe/frame.py", line 1810, in collect
>     return wrap_df(ldf.collect())
>                    ^^^^^^^^^^^^^
> polars.exceptions.ComputeError: conversion from `struct[3]` to `str` failed in column 'failure_case' for 1 out of 1 values: [{"abc","f",null}]

```

#### Expected behavior
It should work with column that have null and are set **nullable=True**

#### versions
pandera: 0.19.0b3
polars: 0.20.23
python: 3.11



Discussion:
I'm not a pandera user - but this is my understanding of why it is failing:

It seems the `failure_case` column can be a string or a struct.

In the case of a struct, this fails:

https://github.com/unionai-oss/pandera/blob/dbf18314fc9461b3f7af8ff6c4741a6dff0f99ac/pandera/backends/polars/base.py#L173-L175

```python
import polars as pl

df = pl.DataFrame({
    'failure_case': [{'case_id': 'abc', 'extract_date': None}]
})

df.with_columns(pl.col("failure_case").cast(pl.String))
# ComputeError: conversion from `struct[2]` to `str` failed in column ...
```

A struct can be "stringified" in Polars via [`.struct.json_encode()`](https://docs.pola.rs/docs/python/dev/reference/expressions/api/polars.Expr.struct.json_encode.html)

```python
>>> df.with_columns(pl.col("failure_case").struct.json_encode())
shape: (1, 1)
┌───────────────────────────────────────┐
│ failure_case                          │
│ ---                                   │
│ str                                   │
╞═══════════════════════════════════════╡
│ {"case_id":"abc","extract_date":null} │
└───────────────────────────────────────┘
```

But I'm not sure if that's what pandera wants to do in this case.

Resolving patch:
diff --git a/pandera/api/polars/model.py b/pandera/api/polars/model.py
--- a/pandera/api/polars/model.py
+++ b/pandera/api/polars/model.py
@@ -57,7 +57,7 @@ def _build_columns(  # pylint:disable=too-many-locals
             try:
                 engine_dtype = pe.Engine.dtype(annotation.raw_annotation)
                 dtype = engine_dtype.type
-            except TypeError as exc:
+            except (TypeError, ValueError) as exc:
                 if annotation.metadata:
                     if field.dtype_kwargs:
                         raise TypeError(
diff --git a/pandera/backends/polars/base.py b/pandera/backends/polars/base.py
--- a/pandera/backends/polars/base.py
+++ b/pandera/backends/polars/base.py
@@ -158,7 +158,7 @@ def failure_cases_metadata(
                         failure_case=pl.Series(
                             err.failure_cases.rows(named=True)
                         )
-                    ).select(pl.col.failure_case)
+                    ).select(pl.col.failure_case.struct.json_encode())
                 else:
                     failure_cases_df = err.failure_cases.rename(
                         {err.failure_cases.columns[0]: "failure_case"}
diff --git a/pandera/engines/polars_engine.py b/pandera/engines/polars_engine.py
--- a/pandera/engines/polars_engine.py
+++ b/pandera/engines/polars_engine.py
@@ -156,6 +156,8 @@ def try_coerce(self, data_container: PolarsDataContainer) -> pl.LazyFrame:
             is_coercible, failure_cases = polars_coerce_failure_cases(
                 data_container=data_container, type_=self.type
             )
+            if data_container.key:
+                failure_cases = failure_cases.select(data_container.key)
             raise errors.ParserError(
                 f"Could not coerce {_key} LazyFrame with schema "
                 f"{data_container.lazyframe.schema} "


Test patch:
diff --git a/tests/polars/test_polars_config.py b/tests/polars/test_polars_config.py
--- a/tests/polars/test_polars_config.py
+++ b/tests/polars/test_polars_config.py
@@ -171,6 +171,4 @@ def test_coerce_validation_depth_none(validation_depth_none, schema):
         try:
             schema.validate(data)
         except pa.errors.SchemaError as exc:
-            assert exc.failure_cases.rows(named=True) == [
-                {"a": "foo", "b": "c"}
-            ]
+            assert exc.failure_cases.rows(named=True) == [{"a": "foo"}]
diff --git a/tests/polars/test_polars_container.py b/tests/polars/test_polars_container.py
--- a/tests/polars/test_polars_container.py
+++ b/tests/polars/test_polars_container.py
@@ -564,3 +564,28 @@ def test_dataframe_schema_with_tz_agnostic_dates(time_zone, data):
     if time_zone:
         with pytest.raises(pa.errors.SchemaError):
             schema_tz_sensitive.validate(lf)
+
+
+def test_dataframe_coerce_col_with_null_in_other_column():
+    class Model(DataFrameModel):
+        col1: int = pa.Field(nullable=False, coerce=True)
+        col2: float = pa.Field(nullable=True, coerce=True)
+
+    invalid_lf = pl.DataFrame(
+        {
+            "col1": ["1", "2", "abc"],
+            "col2": [1.0, 2.0, None],
+        }
+    )
+
+    try:
+        print(Model.validate(invalid_lf, lazy=True))
+    except pa.errors.SchemaErrors as exc:
+        failures = exc.failure_cases.select("failure_case").rows(named=True)
+        # two failures should occur:
+        # - Coercing "abc" to int
+        # - Validating that col1 is an integer
+        assert failures == [
+            {"failure_case": "abc"},
+            {"failure_case": "String"},
+        ]