Repository: duckdb/duckdb (language: python)

Issue:
WHERE clause ignores Decimal comparison when using Polars DataFrame
### What happens?

When filtering a Polars DataFrame containing Decimal values using `WHERE column > value`, DuckDB includes rows that fail the comparison, but this only happens with Polars DataFrames - not with Parquet files or DuckDB tables containing identical data.

### To Reproduce

```python
import duckdb
import polars as pl
import numpy as np
from pathlib import Path
import tempfile

np.random.seed(10)

with tempfile.TemporaryDirectory() as tmp_dir:
    path = Path(tmp_dir) / "test.parquet"
    pl.DataFrame({'x': pl.Series(np.random.uniform(-10, 10, 1000)).cast(pl.Decimal(18, 4))}).write_parquet(path, use_pyarrow=True)
    
    conn = duckdb.connect()
    conn.sql(f"CREATE TABLE x_table AS SELECT * FROM '{path}'")

    df_pl = pl.read_parquet(path)

    query = """
    SELECT x, x > 0.05 AS is_x_good, x::FLOAT > 0.05 AS is_float_x_good
    FROM {}
    WHERE is_x_good
    ORDER BY x ASC
    """
    print("DuckDB table:\n", conn.sql(query.format('x_table')), sep='')
    print("Parquet from DuckDB:\n", conn.sql(query.format(f"read_parquet('{path}')")), sep='')
    print("Polars df from DuckDB:\n", conn.sql(query.format("df_pl")), sep='')

    query_cte = """
    WITH base AS(
        SELECT x, x > 0.05 AS is_x_good, x::FLOAT > 0.05 AS is_float_x_good
        FROM df_pl
    )
    SELECT *
    FROM base
    WHERE is_x_good
    ORDER BY x ASC
    """
    print("Polars df from DuckDB cte:\n", conn.sql(query_cte), sep='')
```

#### Expected behavior
Querying the Polars DataFrame should return the same results as querying the Parquet file or DuckDB table directly - rows where `x > 0.05` evaluates to `true`.

#### Actual behavior

Full output showing the discrepancy:

```
DuckDB table:
┌───────────────┬───────────┬─────────────────┐
│       x       │ is_x_good │ is_float_x_good │
│ decimal(18,4) │  boolean  │     boolean     │
├───────────────┼───────────┼─────────────────┤
│        0.0533 │ true      │ true            │
│        0.0714 │ true      │ true            │
│        0.0901 │ true      │ true            │
│        0.0968 │ true      │ true            │
│        0.1011 │ true      │ true            │
│        0.1395 │ true      │ true            │
│        0.1520 │ true      │ true            │
│        0.1867 │ true      │ true            │
│        0.2066 │ true      │ true            │
│        0.2126 │ true      │ true            │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│        9.7783 │ true      │ true            │
│        9.8024 │ true      │ true            │
│        9.8033 │ true      │ true            │
│        9.8187 │ true      │ true            │
│        9.8965 │ true      │ true            │
│        9.9051 │ true      │ true            │
│        9.9246 │ true      │ true            │
│        9.9409 │ true      │ true            │
│        9.9615 │ true      │ true            │
│        9.9727 │ true      │ true            │
├───────────────┴───────────┴─────────────────┤
│ 495 rows (20 shown)               3 columns │
└─────────────────────────────────────────────┘

Parquet from DuckDB:
┌───────────────┬───────────┬─────────────────┐
│       x       │ is_x_good │ is_float_x_good │
│ decimal(18,4) │  boolean  │     boolean     │
├───────────────┼───────────┼─────────────────┤
│        0.0533 │ true      │ true            │
│        0.0714 │ true      │ true            │
│        0.0901 │ true      │ true            │
│        0.0968 │ true      │ true            │
│        0.1011 │ true      │ true            │
│        0.1395 │ true      │ true            │
│        0.1520 │ true      │ true            │
│        0.1867 │ true      │ true            │
│        0.2066 │ true      │ true            │
│        0.2126 │ true      │ true            │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│        9.7783 │ true      │ true            │
│        9.8024 │ true      │ true            │
│        9.8033 │ true      │ true            │
│        9.8187 │ true      │ true            │
│        9.8965 │ true      │ true            │
│        9.9051 │ true      │ true            │
│        9.9246 │ true      │ true            │
│        9.9409 │ true      │ true            │
│        9.9615 │ true      │ true            │
│        9.9727 │ true      │ true            │
├───────────────┴───────────┴─────────────────┤
│ 495 rows (20 shown)               3 columns │
└─────────────────────────────────────────────┘

Polars df from DuckDB:
┌───────────────┬───────────┬─────────────────┐
│       x       │ is_x_good │ is_float_x_good │
│ decimal(18,4) │  boolean  │     boolean     │
├───────────────┼───────────┼─────────────────┤
│        0.0349 │ false     │ false           │  <- Should not appear
│        0.0404 │ false     │ false           │  <- Should not appear
│        0.0533 │ true      │ true            │
│        0.0714 │ true      │ true            │
│        0.0901 │ true      │ true            │
│        0.0968 │ true      │ true            │
│        0.1011 │ true      │ true            │
│        0.1395 │ true      │ true            │
│        0.1520 │ true      │ true            │
│        0.1867 │ true      │ true            │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│        9.7783 │ true      │ true            │
│        9.8024 │ true      │ true            │
│        9.8033 │ true      │ true            │
│        9.8187 │ true      │ true            │
│        9.8965 │ true      │ true            │
│        9.9051 │ true      │ true            │
│        9.9246 │ true      │ true            │
│        9.9409 │ true      │ true            │
│        9.9615 │ true      │ true            │
│        9.9727 │ true      │ true            │
├───────────────┴───────────┴─────────────────┤
│ 497 rows (20 shown)               3 columns │
└─────────────────────────────────────────────┘

Polars df from DuckDB cte:
┌───────────────┬───────────┬─────────────────┐
│       x       │ is_x_good │ is_float_x_good │
│ decimal(18,4) │  boolean  │     boolean     │
├───────────────┼───────────┼─────────────────┤
│        0.0349 │ false     │ false           │  <- Should not appear
│        0.0404 │ false     │ false           │  <- Should not appear
│        0.0533 │ true      │ true            │
│        0.0714 │ true      │ true            │
│        0.0901 │ true      │ true            │
│        0.0968 │ true      │ true            │
│        0.1011 │ true      │ true            │
│        0.1395 │ true      │ true            │
│        0.1520 │ true      │ true            │
│        0.1867 │ true      │ true            │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│           ·   │  ·        │  ·              │
│        9.7783 │ true      │ true            │
│        9.8024 │ true      │ true            │
│        9.8033 │ true      │ true            │
│        9.8187 │ true      │ true            │
│        9.8965 │ true      │ true            │
│        9.9051 │ true      │ true            │
│        9.9246 │ true      │ true            │
│        9.9409 │ true      │ true            │
│        9.9615 │ true      │ true            │
│        9.9727 │ true      │ true            │
├───────────────┴───────────┴─────────────────┤
│ 497 rows (20 shown)               3 columns │
└─────────────────────────────────────────────┘
```

When querying the Polars DataFrame, rows where both `is_x_good` and `is_float_x_good` are `false` appear in the results, despite the `WHERE is_x_good` clause. This doesn't happen when querying the Parquet file or DuckDB table directly.

Interestingly, filtering on `WHERE is_float_x_good` (casting to FLOAT first) produces the expected results.

DuckDB 1.1.1
Polars 1.7.1


### OS:

Linux

### DuckDB Version:

1.1.1

### DuckDB Client:

Python

### Hardware:

_No response_

### Full Name:

Pavel Khokhlov

### Affiliation:

personal

### What is the latest build you tested with? If possible, we recommend testing with the latest nightly build.

I have tested with a stable release

### Did you include all relevant data sets for reproducing the issue?

Yes

### Did you include all code required to reproduce the issue?

- [X] Yes, I have

### Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue?

- [X] Yes, I have


Discussion / hints:

Fix patch:
diff --git a/tools/pythonpkg/duckdb_extension_config.cmake b/tools/pythonpkg/duckdb_extension_config.cmake
index 38622df8cc84..0c978737d879 100644
--- a/tools/pythonpkg/duckdb_extension_config.cmake
+++ b/tools/pythonpkg/duckdb_extension_config.cmake
@@ -7,7 +7,6 @@
 # CMakeLists.txt file with the `BUILD_PYTHON` variable.
 # TODO: unify this by making setup.py also use this configuration, making this the config for all python builds
 duckdb_extension_load(json)
-duckdb_extension_load(fts)
 duckdb_extension_load(tpcds)
 duckdb_extension_load(tpch)
 duckdb_extension_load(parquet)
diff --git a/tools/pythonpkg/src/arrow/arrow_array_stream.cpp b/tools/pythonpkg/src/arrow/arrow_array_stream.cpp
index 1fc37f1195bc..47700fb24d6a 100644
--- a/tools/pythonpkg/src/arrow/arrow_array_stream.cpp
+++ b/tools/pythonpkg/src/arrow/arrow_array_stream.cpp
@@ -276,25 +276,13 @@ py::object GetScalar(Value &constant, const string &timezone_config, const Arrow
 	case LogicalTypeId::BLOB:
 		return dataset_scalar(py::bytes(constant.GetValueUnsafe<string>()));
 	case LogicalTypeId::DECIMAL: {
-		py::object date_type = py::module_::import("pyarrow").attr("decimal128");
+		py::object decimal_type = py::module_::import("pyarrow").attr("decimal128");
 		uint8_t width;
 		uint8_t scale;
 		constant.type().GetDecimalProperties(width, scale);
-		switch (constant.type().InternalType()) {
-		case PhysicalType::INT16:
-			return dataset_scalar(scalar(constant.GetValue<int16_t>(), date_type(width, scale)));
-		case PhysicalType::INT32:
-			return dataset_scalar(scalar(constant.GetValue<int32_t>(), date_type(width, scale)));
-		case PhysicalType::INT64:
-			return dataset_scalar(scalar(constant.GetValue<int64_t>(), date_type(width, scale)));
-		default: {
-			auto hugeint_value = constant.GetValue<hugeint_t>();
-			auto hugeint_value_py = py::cast(hugeint_value.upper);
-			hugeint_value_py = hugeint_value_py.attr("__mul__")(NumericLimits<uint64_t>::Maximum());
-			hugeint_value_py = hugeint_value_py.attr("__add__")(hugeint_value.lower);
-			return dataset_scalar(scalar(hugeint_value_py, date_type(width, scale)));
-		}
-		}
+		// pyarrow only allows 'decimal.Decimal' to be used to construct decimal scalars such as 0.05
+		auto val = import_cache.decimal.Decimal()(constant.ToString());
+		return dataset_scalar(scalar(std::move(val), decimal_type(width, scale)));
 	}
 	default:
 		throw NotImplementedException("Unimplemented type \"%s\" for Arrow Filter Pushdown",


Test patch:
diff --git a/tools/pythonpkg/tests/fast/arrow/test_filter_pushdown.py b/tools/pythonpkg/tests/fast/arrow/test_filter_pushdown.py
index f03d8e561b45..5cc467f6c6e8 100644
--- a/tools/pythonpkg/tests/fast/arrow/test_filter_pushdown.py
+++ b/tools/pythonpkg/tests/fast/arrow/test_filter_pushdown.py
@@ -32,6 +32,27 @@ def create_pyarrow_dataset(rel):
     return ds.dataset(table)
 
 
+def test_decimal_filter_pushdown(duckdb_cursor):
+    pl = pytest.importorskip("polars")
+    np = pytest.importorskip("numpy")
+    np.random.seed(10)
+
+    df = pl.DataFrame({'x': pl.Series(np.random.uniform(-10, 10, 1000)).cast(pl.Decimal(18, 4))})
+
+    query = """
+        SELECT
+            x,
+            x > 0.05 AS is_x_good,
+            x::FLOAT > 0.05 AS is_float_x_good
+        FROM {}
+        WHERE
+            is_x_good
+        ORDER BY x ASC
+    """
+
+    assert len(duckdb_cursor.sql(query.format("df")).fetchall()) == 495
+
+
 def numeric_operators(connection, data_type, tbl_name, create_table):
     connection.execute(
         f"""