Repository: duckdb/duckdb (language: python)

Issue:
pd.NA values get coerced to 0.0 for a pd.Float64DType 
### What happens?

I have a Pandas dataframe that I load from a Parquet file. If a column in that file is of pd.Float64DType, DuckDB does not convert pd.NA values to `NULL`, which would be an expected behaviour. 

### To Reproduce

```
import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import duckdb
```

```
testdf = pd.DataFrame({"value": [26.0, 26.0, 26.0, pd.NA, 27.0, pd.NA, pd.NA, pd.NA, pd.NA, 29.0]})
testdf["value"] = testdf["value"].astype(pd.Float64Dtype())
```

```
pq.write_table(pa.Table.from_pandas(testdf), "./testdf.parquet")
```

```
testdf2 = pd.read_parquet("./testdf.parquet")
```

```
con = duckdb.connect(database=":memory:")
con.register("testdf", testdf)
con.register("testdf2", testdf2)
```

```
con.sql("SELECT MIN(value) FROM testdf")
```
```
┌──────────────┐
│ min("value") │
│    double    │
├──────────────┤
│         26.0 │
└──────────────┘
```

```
testdf2["value"].min()
```

```
26.0
```

```
con.sql("SELECT MIN(value) FROM testdf2")
```

```
┌──────────────┐
│ min("value") │
│    double    │
├──────────────┤
│          0.0 │
└──────────────┘
```

### OS:

MacOS, x86_64

### DuckDB Version:

1.1.1.

### DuckDB Client:

Python

### Hardware:

_No response_

### Full Name:

Dmitry Kryuchkov

### Affiliation:

SHERLOK TECHNOLOGY PTY LTD

### What is the latest build you tested with? If possible, we recommend testing with the latest nightly build.

I have tested with a stable release

### Did you include all relevant data sets for reproducing the issue?

Yes

### Did you include all code required to reproduce the issue?

- [X] Yes, I have

### Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue?

- [X] Yes, I have


Discussion / hints:
Float64DType is not a parquet data type. Given how bad pandas is around nullability, I wouldn't be surprised if it's a problem on their end (e.g. during the conversion to pyarrow). Did you try reading the parquet with a third party tool? Did you try writing from pandas to parquet directly without explicitly pyarrow conversion?

Fix patch:
diff --git a/tools/pythonpkg/src/numpy/numpy_scan.cpp b/tools/pythonpkg/src/numpy/numpy_scan.cpp
index 54563204de5c..90f061416a83 100644
--- a/tools/pythonpkg/src/numpy/numpy_scan.cpp
+++ b/tools/pythonpkg/src/numpy/numpy_scan.cpp
@@ -59,25 +59,31 @@ void ScanNumpyCategory(py::array &column, idx_t count, idx_t offset, Vector &out
 	}
 }
 
+static void ApplyMask(PandasColumnBindData &bind_data, ValidityMask &validity, idx_t count, idx_t offset) {
+	D_ASSERT(bind_data.mask);
+	auto mask = reinterpret_cast<const bool *>(bind_data.mask->numpy_array.data());
+	for (idx_t i = 0; i < count; i++) {
+		auto is_null = mask[offset + i];
+		if (is_null) {
+			validity.SetInvalid(i);
+		}
+	}
+}
+
 template <class T>
 void ScanNumpyMasked(PandasColumnBindData &bind_data, idx_t count, idx_t offset, Vector &out) {
 	D_ASSERT(bind_data.pandas_col->Backend() == PandasColumnBackend::NUMPY);
 	auto &numpy_col = reinterpret_cast<PandasNumpyColumn &>(*bind_data.pandas_col);
 	ScanNumpyColumn<T>(numpy_col.array, numpy_col.stride, offset, out, count);
-	auto &result_mask = FlatVector::Validity(out);
 	if (bind_data.mask) {
-		auto mask = reinterpret_cast<const bool *>(bind_data.mask->numpy_array.data());
-		for (idx_t i = 0; i < count; i++) {
-			auto is_null = mask[offset + i];
-			if (is_null) {
-				result_mask.SetInvalid(i);
-			}
-		}
+		auto &result_mask = FlatVector::Validity(out);
+		ApplyMask(bind_data, result_mask, count, offset);
 	}
 }
 
 template <class T>
-void ScanNumpyFpColumn(const T *src_ptr, idx_t stride, idx_t count, idx_t offset, Vector &out) {
+void ScanNumpyFpColumn(PandasColumnBindData &bind_data, const T *src_ptr, idx_t stride, idx_t count, idx_t offset,
+                       Vector &out) {
 	auto &mask = FlatVector::Validity(out);
 	if (stride == sizeof(T)) {
 		FlatVector::SetData(out, (data_ptr_t)(src_ptr + offset)); // NOLINT
@@ -97,6 +103,10 @@ void ScanNumpyFpColumn(const T *src_ptr, idx_t stride, idx_t count, idx_t offset
 			}
 		}
 	}
+	if (bind_data.mask) {
+		auto &result_mask = FlatVector::Validity(out);
+		ApplyMask(bind_data, result_mask, count, offset);
+	}
 }
 
 template <class T>
@@ -228,10 +238,12 @@ void NumpyScan::Scan(PandasColumnBindData &bind_data, idx_t count, idx_t offset,
 		ScanNumpyMasked<int64_t>(bind_data, count, offset, out);
 		break;
 	case NumpyNullableType::FLOAT_32:
-		ScanNumpyFpColumn<float>(reinterpret_cast<const float *>(array.data()), numpy_col.stride, count, offset, out);
+		ScanNumpyFpColumn<float>(bind_data, reinterpret_cast<const float *>(array.data()), numpy_col.stride, count,
+		                         offset, out);
 		break;
 	case NumpyNullableType::FLOAT_64:
-		ScanNumpyFpColumn<double>(reinterpret_cast<const double *>(array.data()), numpy_col.stride, count, offset, out);
+		ScanNumpyFpColumn<double>(bind_data, reinterpret_cast<const double *>(array.data()), numpy_col.stride, count,
+		                          offset, out);
 		break;
 	case NumpyNullableType::DATETIME_NS:
 	case NumpyNullableType::DATETIME_MS:


Test patch:
diff --git a/tools/pythonpkg/tests/fast/pandas/test_pandas_types.py b/tools/pythonpkg/tests/fast/pandas/test_pandas_types.py
index 124b296eb33e..b5f5d6327bec 100644
--- a/tools/pythonpkg/tests/fast/pandas/test_pandas_types.py
+++ b/tools/pythonpkg/tests/fast/pandas/test_pandas_types.py
@@ -1,4 +1,5 @@
 import duckdb
+import pytest
 import pandas as pd
 import numpy
 import string
@@ -74,6 +75,27 @@ def test_pandas_bool(self, duckdb_cursor):
         data = numpy.array([True, False, False, True])
         round_trip(data, 'bool')
 
+    def test_pandas_masked_float64(self, duckdb_cursor, tmp_path):
+        pa = pytest.importorskip("pyarrow")
+        pq = pytest.importorskip("pyarrow.parquet")
+
+        # Create a sample DataFrame
+        testdf = pd.DataFrame({"value": [26.0, 26.0, 26.0, pd.NA, 27.0, pd.NA, pd.NA, pd.NA, pd.NA, 29.0]})
+
+        # Set the correct dtype for the 'value' column
+        testdf["value"] = testdf["value"].astype(pd.Float64Dtype())
+
+        # Write the DataFrame to a Parquet file using tmp_path fixture
+        parquet_path = tmp_path / "testdf.parquet"
+        pq.write_table(pa.Table.from_pandas(testdf), parquet_path)
+
+        # Read the Parquet file back into a DataFrame
+        testdf2 = pd.read_parquet(parquet_path)
+
+        # Use duckdb_cursor to query the parquet data
+        result = duckdb_cursor.execute("SELECT MIN(value) FROM testdf2").fetchall()
+        assert result[0][0] == 26
+
     def test_pandas_boolean(self, duckdb_cursor):
         data = numpy.array([True, None, pd.NA, numpy.nan, True])
         df_in = pd.DataFrame(