Repository: duckdb/duckdb (language: python)

Issue:
NULL pointer dereference scanning from arrow with lists of structs
### What happens?

There's a segfault when trying to select a column from an arrow batch that has two lists and one of them is a list of structs. Maybe there's a simpler explanation, but this is what I've been able to trivially reproduce. I first filed this under https://github.com/marcboeker/go-duckdb/issues/321, but it's not very go specific, and the Python support seems more first-class so this repro seemed more likely to get some attention.

### To Reproduce

Create an arrow batch with lists of structs and then try to scan one of them. Below is an example in Python, but it also happens using the C-API from go. 

```python
# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "duckdb",
#     "pyarrow",
# ]
# ///

import pyarrow as pa

schema = pa.schema([
    pa.field("m", pa.struct([
        pa.field("array", pa.list_(
            pa.struct([pa.field("a", pa.int64())])
        ))
    ])),
    pa.field("a", pa.struct([
        pa.field("array", pa.list_(pa.int64()))
    ]))
])
m_a_values = pa.array([1], type=pa.int64())  # Values for "a"
m_array = pa.ListArray.from_arrays(
    offsets=pa.array([0, 1], type=pa.int32()),  # One list with one element
    values=pa.StructArray.from_arrays([m_a_values], names=["a"])
)
m = pa.StructArray.from_arrays([m_array], names=["array"])
a_array_values = pa.array([1], type=pa.int64())  # Values for the list
a_array = pa.ListArray.from_arrays(
    offsets=pa.array([0, 1], type=pa.int32()),  # One list with one element
    values=a_array_values
)
a = pa.StructArray.from_arrays([a_array], names=["array"])
record_batch = pa.RecordBatch.from_arrays([m, a], schema.names)

import duckdb
con = duckdb.connect()
results = con.execute("SELECT a FROM record_batch").arrow()
```

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
duckdb.duckdb.InternalException: INTERNAL Error: Attempted to dereference unique_ptr that is NULL!
This error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.
For more information, see https://duckdb.org/docs/dev/internal_errors
```

### OS:

linux x86_64

### DuckDB Version:

1.1.3

### DuckDB Client:

Python

### Hardware:

_No response_

### Full Name:

Andrew Werner

### Affiliation:

Data Ex Machina, Inc

### What is the latest build you tested with? If possible, we recommend testing with the latest nightly build.

I have tested with a stable release

### Did you include all relevant data sets for reproducing the issue?

Yes

### Did you include all code required to reproduce the issue?

- [X] Yes, I have

### Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue?

- [X] Yes, I have


Discussion / hints:
I'll add that `select *` works, but many other subselects such as `select a` , `select m`, `select a.*`, `select a.array`, etc lead to the crash. 

Fix patch:
diff --git a/src/function/table/arrow.cpp b/src/function/table/arrow.cpp
index f5bea5a3abbe..a42dfc6a703e 100644
--- a/src/function/table/arrow.cpp
+++ b/src/function/table/arrow.cpp
@@ -370,6 +370,15 @@ void ArrowTableFunction::PopulateArrowTableType(ArrowTableType &arrow_table, Arr
 	}
 }
 
+unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBindDumb(ClientContext &context, TableFunctionBindInput &input,
+                                                               vector<LogicalType> &return_types,
+                                                               vector<string> &names) {
+	auto bind_data = ArrowScanBind(context, input, return_types, names);
+	auto &arrow_bind_data = bind_data->Cast<ArrowScanFunctionData>();
+	arrow_bind_data.projection_pushdown_enabled = false;
+	return bind_data;
+}
+
 unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
                                                            vector<LogicalType> &return_types, vector<string> &names) {
 	if (input.inputs[0].IsNull() || input.inputs[1].IsNull() || input.inputs[2].IsNull()) {
@@ -475,7 +484,10 @@ ArrowTableFunction::ArrowScanInitLocalInternal(ClientContext &context, TableFunc
 	auto result = make_uniq<ArrowScanLocalState>(std::move(current_chunk));
 	result->column_ids = input.column_ids;
 	result->filters = input.filters.get();
-	if (!input.projection_ids.empty()) {
+	auto &bind_data = input.bind_data->Cast<ArrowScanFunctionData>();
+	if (!bind_data.projection_pushdown_enabled) {
+		result->column_ids.clear();
+	} else if (!input.projection_ids.empty()) {
 		auto &asgs = global_state_p->Cast<ArrowScanGlobalState>();
 		result->all_columns.Initialize(context, asgs.scanned_types);
 	}
@@ -594,7 +606,7 @@ void ArrowTableFunction::RegisterFunction(BuiltinFunctions &set) {
 	set.AddFunction(arrow);
 
 	TableFunction arrow_dumb("arrow_scan_dumb", {LogicalType::POINTER, LogicalType::POINTER, LogicalType::POINTER},
-	                         ArrowScanFunction, ArrowScanBind, ArrowScanInitGlobal, ArrowScanInitLocal);
+	                         ArrowScanFunction, ArrowScanBindDumb, ArrowScanInitGlobal, ArrowScanInitLocal);
 	arrow_dumb.cardinality = ArrowScanCardinality;
 	arrow_dumb.get_partition_data = ArrowGetPartitionData;
 	arrow_dumb.projection_pushdown = false;
diff --git a/src/function/table/arrow_conversion.cpp b/src/function/table/arrow_conversion.cpp
index 77ba98746898..8a19c1b7dc37 100644
--- a/src/function/table/arrow_conversion.cpp
+++ b/src/function/table/arrow_conversion.cpp
@@ -1340,7 +1340,7 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr
 void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data,
                                        DataChunk &output, idx_t start, bool arrow_scan_is_projected) {
 	for (idx_t idx = 0; idx < output.ColumnCount(); idx++) {
-		auto col_idx = scan_state.column_ids[idx];
+		auto col_idx = scan_state.column_ids.empty() ? idx : scan_state.column_ids[idx];
 
 		// If projection was not pushed down into the arrow scanner, but projection pushdown is enabled on the
 		// table function, we need to use original column ids here.
diff --git a/src/include/duckdb/function/table/arrow.hpp b/src/include/duckdb/function/table/arrow.hpp
index d9930f466a74..4f8f24f3d02a 100644
--- a/src/include/duckdb/function/table/arrow.hpp
+++ b/src/include/duckdb/function/table/arrow.hpp
@@ -65,6 +65,8 @@ struct ArrowScanFunctionData : public TableFunctionData {
 	shared_ptr<DependencyItem> dependency;
 	//! Arrow table data
 	ArrowTableType arrow_table;
+	//! Whether projection pushdown is enabled on the scan
+	bool projection_pushdown_enabled = true;
 };
 
 struct ArrowRunEndEncodingState {
@@ -184,6 +186,8 @@ struct ArrowTableFunction {
 	//! Binds an arrow table
 	static unique_ptr<FunctionData> ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
 	                                              vector<LogicalType> &return_types, vector<string> &names);
+	static unique_ptr<FunctionData> ArrowScanBindDumb(ClientContext &context, TableFunctionBindInput &input,
+	                                                  vector<LogicalType> &return_types, vector<string> &names);
 	//! Actual conversion from Arrow to DuckDB
 	static void ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data,
 	                          DataChunk &output, idx_t start, bool arrow_scan_is_projected = true);
diff --git a/tools/pythonpkg/duckdb_extension_config.cmake b/tools/pythonpkg/duckdb_extension_config.cmake
index 38622df8cc84..0c978737d879 100644
--- a/tools/pythonpkg/duckdb_extension_config.cmake
+++ b/tools/pythonpkg/duckdb_extension_config.cmake
@@ -7,7 +7,6 @@
 # CMakeLists.txt file with the `BUILD_PYTHON` variable.
 # TODO: unify this by making setup.py also use this configuration, making this the config for all python builds
 duckdb_extension_load(json)
-duckdb_extension_load(fts)
 duckdb_extension_load(tpcds)
 duckdb_extension_load(tpch)
 duckdb_extension_load(parquet)


Test patch:
diff --git a/tools/pythonpkg/tests/fast/arrow/test_projection_pushdown.py b/tools/pythonpkg/tests/fast/arrow/test_projection_pushdown.py
index 069c4a10112a..39022ab4d752 100644
--- a/tools/pythonpkg/tests/fast/arrow/test_projection_pushdown.py
+++ b/tools/pythonpkg/tests/fast/arrow/test_projection_pushdown.py
@@ -2,27 +2,33 @@
 import os
 import pytest
 
-try:
-    import pyarrow as pa
-    import pyarrow.dataset as ds
-
-    can_run = True
-except:
-    can_run = False
-
 
 class TestArrowProjectionPushdown(object):
     def test_projection_pushdown_no_filter(self, duckdb_cursor):
-        if not can_run:
-            return
-        duckdb_conn = duckdb.connect()
-        duckdb_conn.execute("CREATE TABLE test (a  INTEGER, b INTEGER, c INTEGER)")
-        duckdb_conn.execute("INSERT INTO  test VALUES (1,1,1),(10,10,10),(100,10,100),(NULL,NULL,NULL)")
-        duck_tbl = duckdb_conn.table("test")
+        pa = pytest.importorskip("pyarrow")
+        ds = pytest.importorskip("pyarrow.dataset")
+
+        duckdb_cursor.execute(
+            """
+            CREATE TABLE test (a  INTEGER, b INTEGER, c INTEGER)
+        """
+        )
+        duckdb_cursor.execute(
+            """
+            INSERT INTO test VALUES
+                (1,2,3),
+                (10,20,30),
+                (100,200,300),
+                (NULL,NULL,NULL)
+        """
+        )
+        duck_tbl = duckdb_cursor.table("test")
         arrow_table = duck_tbl.arrow()
-        duckdb_conn.register("testarrowtable", arrow_table)
-        assert duckdb_conn.execute("SELECT sum(a) FROM  testarrowtable").fetchall() == [(111,)]
+        assert duckdb_cursor.execute("SELECT sum(c) FROM arrow_table").fetchall() == [(333,)]
+
+        # RecordBatch does not use projection pushdown, test that this also still works
+        record_batch = arrow_table.to_batches()[0]
+        assert duckdb_cursor.execute("SELECT sum(c) FROM record_batch").fetchall() == [(333,)]
 
         arrow_dataset = ds.dataset(arrow_table)
-        duckdb_conn.register("testarrowdataset", arrow_dataset)
-        assert duckdb_conn.execute("SELECT sum(a) FROM  testarrowdataset").fetchall() == [(111,)]
+        assert duckdb_cursor.execute("SELECT sum(c) FROM arrow_dataset").fetchall() == [(333,)]