Repository: unionai-oss/pandera

Issue:
Datasets that fail DataFrameModel validation unexpectedly alter field properties
**Describe the bug**
When a dataframe fails validation, it can corrupts the state of the `coerce` attribute for a particular field.

In my case, a field is defined as coerce=True, but after trying to validate the bad dataset, the field now has coerce=False.




- [x] I have checked that this issue has not already been reported.
- [x] I have confirmed this bug exists on the latest version of pandera.
- [ ] (optional) I have confirmed this bug exists on the main branch of pandera.

**Note**: Please read [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) detailing how to provide the necessary information for us to reproduce your bug.

#### Code Sample, a copy-pastable example

```python
import pandas as pd
import pandera as pa
from pandera.typing import Series

class Table(pa.DataFrameModel):
    """Simple table with 2 columns."""

    chr: Series[str] = pa.Field(nullable=False, description="Chromosome", str_length=dict(min_value=1), coerce=True)
    start: Series[int] = pa.Field(nullable=False, ge=0, description="0-based inclusive start position of region")


assert Table.to_schema().columns["chr"].coerce # Passes as expected


Table.validate(pd.DataFrame({"chr": ["chr1", "chr2"], "start": [0, 10]}))
assert Table.to_schema().columns["chr"].coerce # Still passes as expected


try:
    Table.validate(pd.DataFrame({"chr": ["", "chr1"], "start": [0, 10]}))
    raise AssertionError("Dataframe should fail validation as str_length constraint not met")
except pa.errors.SchemaError:
    ...

# Unexpectedly fails. coerce is now False for this Field.
# Failed validation essentially corrupted the state of the class
assert Table.to_schema().columns["chr"].coerce
```

#### Expected behavior
The state of a DataFrameModel should not be changed by failed dataframe manipulation.

#### Additional context
I believe the problem is caused by these lines: https://github.com/unionai-oss/pandera/blob/main/pandera/backends/pandas/container.py#L221-L223

The coerce attribute of the field is being changed during dataframe validation, and is intended to be restored once validation completes. But if an exception is raised, the attributes are not properly reverted as the code just jumps to the except blocks. 

A simple fix is just to move these reversion lines outside of the try-except block, after the last of of the exception blocks (so that reversion is always applied regardless of validation success). Alternatively, perhaps it's cleaner to deepcopy the schema_component to avoid the complicated logic regarding changing and reverting these various attributes.


Discussion:

Patch:
diff --git a/pandera/api/pandas/model.py b/pandera/api/pandas/model.py
index 3975e43..91658b5 100644
--- a/pandera/api/pandas/model.py
+++ b/pandera/api/pandas/model.py
@@ -214,7 +214,13 @@ class DataFrameModel(_DataFrameModel[pd.DataFrame, DataFrameSchema]):
         def _field_json_schema(field):
             return {
                 "type": "array",
-                "items": {"type": field["type"]},
+                "items": {
+                    "type": (
+                        field["type"]
+                        if field["type"] != "any" or "extDtype" not in field
+                        else field["extDtype"]
+                    )
+                },
             }
 
         return {
diff --git a/pandera/backends/pandas/container.py b/pandera/backends/pandas/container.py
index 081b7ca..b46142c 100644
--- a/pandera/backends/pandas/container.py
+++ b/pandera/backends/pandas/container.py
@@ -200,12 +200,12 @@ class DataFrameSchemaBackend(PandasSchemaBackend):
         check_passed = []
         # schema-component-level checks
         for schema_component in schema_components:
-            try:
-                # make sure the schema component mutations are reverted after
-                # validation
-                _orig_dtype = schema_component.dtype
-                _orig_coerce = schema_component.coerce
+            # make sure the schema component mutations are reverted after
+            # validation
+            _orig_dtype = schema_component.dtype
+            _orig_coerce = schema_component.coerce
 
+            try:
                 if schema.dtype is not None:
                     # override column dtype with dataframe dtype
                     schema_component.dtype = schema.dtype  # type: ignore
@@ -218,10 +218,6 @@ class DataFrameSchemaBackend(PandasSchemaBackend):
                     check_obj, lazy=lazy, inplace=True
                 )
 
-                # revert the schema component mutations
-                schema_component.dtype = _orig_dtype
-                schema_component.coerce = _orig_coerce
-
                 check_passed.append(is_table(result))
             except SchemaError as err:
                 check_results.append(
@@ -244,6 +240,11 @@ class DataFrameSchemaBackend(PandasSchemaBackend):
                         for schema_error in err.schema_errors
                     ]
                 )
+            finally:
+                # revert the schema component mutations
+                schema_component.dtype = _orig_dtype
+                schema_component.coerce = _orig_coerce
+
         assert all(check_passed)
         return check_results
 


Test patch:
diff --git a/tests/core/test_pydantic.py b/tests/core/test_pydantic.py
index cf1755b..dc7c3c1 100644
--- a/tests/core/test_pydantic.py
+++ b/tests/core/test_pydantic.py
@@ -180,3 +180,31 @@ def test_invalid_seriesschema():
 
     with pytest.raises(TypeError if PYDANTIC_V2 else ValidationError):
         SeriesSchemaPydantic(pa_index="1")
+
+
+@pytest.mark.parametrize(
+    "col_type,dtype,item",
+    [
+        (pa.STRING, "string", "hello"),
+        (pa.UINT8, "UInt8", 1),
+        (pa.INT8, "Int8", 1),
+        (pa.BOOL, "boolean", True),
+    ],
+)
+def test_model_with_extensiondtype_column(col_type, dtype, item):
+    """Test that a model with an external dtype is recognized by pydantic."""
+
+    class ExtensionDtypeModel(pa.DataFrameModel):
+        a: Series[col_type]
+
+    class PydanticModel(BaseModel):
+        df: DataFrame[ExtensionDtypeModel]
+
+    assert isinstance(
+        PydanticModel(
+            df=DataFrame[ExtensionDtypeModel](
+                pd.DataFrame({"a": [item]}, dtype=dtype)
+            )
+        ),
+        PydanticModel,
+    )
diff --git a/tests/core/test_schemas.py b/tests/core/test_schemas.py
index b3ac4eb..cca335c 100644
--- a/tests/core/test_schemas.py
+++ b/tests/core/test_schemas.py
@@ -2477,6 +2477,33 @@ def test_schema_coerce_with_regex() -> None:
     assert isinstance(schema_with_regex.validate(df), pd.DataFrame)
 
 
+def test_schema_coerce_preserve_value():
+    """Test that coercing an invalid data preserves the original coerce value."""
+
+    schema = DataFrameSchema(
+        {
+            "chr": Column(
+                str, checks=Check.str_length(min_value=1), coerce=True
+            ),
+            "start": Column(int, checks=Check.ge(0)),
+        }
+    )
+    assert schema.columns["chr"].coerce
+
+    schema.validate(pd.DataFrame({"chr": ["chr1", "chr2"], "start": [0, 10]}))
+    assert schema.columns["chr"].coerce
+
+    try:
+        schema.validate(pd.DataFrame({"chr": ["", "chr1"], "start": [0, 10]}))
+        raise AssertionError(
+            "Dataframe should fail validation as str_length constraint not met"
+        )
+    except errors.SchemaError:
+        ...
+
+    assert schema.columns["chr"].coerce
+
+
 @pytest.mark.parametrize(
     "schema, obj, expected_obj",
     [