Repository: delta-io/delta-rs (2026-04-11T23:13:07Z)

Issue:
Allow for a nested column being passed into Z-Order
# Description
When having nested fields in schema, Z-Order op does not recognize nested columns.

I believe this happens due to the fact that [fields](https://github.com/delta-io/delta-rs/blob/main/crates/deltalake-core/src/operations/optimize.rs#L951) have names of top level fields only. I am happy to contribute, but I am curious if there is a preference for implementing discovery and access of nested fields within `StructType` or within `build_zorder_plan`?

**Use Case**

```python
import deltalake as dl
import pyarrow as pa

schema = pa.schema([
    ("meta", pa.struct([("field_a", pa.int32())])),
])
table = pa.Table.from_pylist(
    [
        {"meta": {"field_a": val}}
        for val in range(0, 100)
    ],
    schema=schema
)

# This works
dl.write_deltalake(
    "bad_z_order_table",
    data=table,
    mode="append",
)

# This fails - Error from https://github.com/delta-io/delta-rs/blob/main/crates/deltalake-core/src/operations/optimize.rs#L961
dl.DeltaTable("bad_z_order_table").optimize.z_order(
    ["meta.field_a"]
)

# This works? - Haven't teste d if it's actually Z-Ordered
dl.DeltaTable("bad_z_order_table").optimize.z_order(
    ["meta"]
)
```

**Related Issue(s)**
None I think


Discussion:
take



Patch:
diff --git a/crates/core/src/operations/optimize.rs b/crates/core/src/operations/optimize.rs
index a88383a7a..eda00ad82 100644
--- a/crates/core/src/operations/optimize.rs
+++ b/crates/core/src/operations/optimize.rs
@@ -55,7 +55,9 @@ use crate::delta_datafusion::{
 };
 use crate::errors::{DeltaResult, DeltaTableError};
 use crate::kernel::transaction::{CommitBuilder, CommitProperties, DEFAULT_RETRIES, PROTOCOL};
-use crate::kernel::{Action, Add, PartitionsExt, Remove, Version, scalars::ScalarExt};
+use crate::kernel::{
+    Action, Add, DataType, PartitionsExt, Remove, StructType, Version, scalars::ScalarExt,
+};
 use crate::kernel::{EagerSnapshot, resolve_snapshot};
 use crate::logstore::{LogStore, LogStoreRef, ObjectStoreRef};
 use crate::parquet_utils::default_writer_properties;
@@ -726,9 +728,9 @@ impl MergePlan {
         context: Arc<zorder::ZOrderExecContext>,
         table_provider: DeltaTableProvider,
     ) -> Result<BoxStream<'static, Result<RecordBatch, ParquetError>>, DeltaTableError> {
-        use datafusion::common::Column;
+        use datafusion::functions::core::expr_ext::FieldAccessor;
         use datafusion::logical_expr::expr::ScalarFunction;
-        use datafusion::logical_expr::{Expr, ScalarUDF};
+        use datafusion::logical_expr::{Expr, ScalarUDF, ident};
 
         let provider = table_provider.with_files(files.files);
         let df = context.ctx.read_table(Arc::new(provider))?;
@@ -736,7 +738,15 @@ impl MergePlan {
         let cols = context
             .columns
             .iter()
-            .map(|col| Expr::Column(Column::from_qualified_name_ignore_case(col)))
+            .map(|col_name| {
+                let mut segments = col_name.split('.');
+                let first = segments.next().expect("column name cannot be empty");
+                let mut expr = ident(first);
+                for segment in segments {
+                    expr = expr.field(segment);
+                }
+                expr
+            })
             .collect_vec();
         let expr = Expr::ScalarFunction(ScalarFunction::new_udf(
             Arc::new(ScalarUDF::from(zorder::datafusion::ZOrderUDF)),
@@ -1235,6 +1245,31 @@ async fn build_compaction_plan(
     ))
 }
 
+/// Validates that a z-order column path exists in the schema, supporting nested
+/// struct fields via dot notation (e.g., "meta.field_a").
+fn validate_zorder_column(schema: &StructType, column: &str) -> Result<(), DeltaTableError> {
+    let mut segments = column.split('.').peekable();
+    let mut current_struct = schema;
+    while let Some(segment) = segments.next() {
+        let field = current_struct.field(segment).ok_or_else(|| {
+            DeltaTableError::Generic(format!(
+                "Z-order column \"{column}\": field \"{segment}\" not found in schema"
+            ))
+        })?;
+        if segments.peek().is_some() {
+            match field.data_type() {
+                DataType::Struct(inner) => current_struct = inner,
+                _ => {
+                    return Err(DeltaTableError::Generic(format!(
+                        "Z-order column \"{column}\": \"{segment}\" is not a struct type"
+                    )));
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
 async fn build_zorder_plan(
     log_store: &dyn LogStore,
     zorder_columns: Vec<String>,
@@ -1257,19 +1292,8 @@ async fn build_zorder_plan(
             "Z-order columns cannot be partition columns. Found: {zorder_partition_cols:?}"
         )));
     }
-    let field_names = snapshot
-        .schema()
-        .fields()
-        .map(|field| field.name().to_string())
-        .collect_vec();
-    let unknown_columns = zorder_columns
-        .iter()
-        .filter(|col| !field_names.contains(col))
-        .collect_vec();
-    if !unknown_columns.is_empty() {
-        return Err(DeltaTableError::Generic(format!(
-            "Z-order columns must be present in the table schema. Unknown columns: {unknown_columns:?}"
-        )));
+    for col in &zorder_columns {
+        validate_zorder_column(snapshot.schema().as_ref(), col)?;
     }
 
     // For now, just be naive and optimize all files in each selected partition.


Test patch:
diff --git a/crates/core/tests/command_optimize.rs b/crates/core/tests/command_optimize.rs
index a249bdf88..ab4db9f96 100644
--- a/crates/core/tests/command_optimize.rs
+++ b/crates/core/tests/command_optimize.rs
@@ -1299,9 +1299,12 @@ async fn test_zorder_rejects_nonexistent_columns() -> Result<(), Box<dyn Error>>
         .with_type(OptimizeType::ZOrder(vec!["non-existent".to_string()]))
         .await;
     assert!(result.is_err());
-    assert!(result.unwrap_err().to_string().contains(
-        "Z-order columns must be present in the table schema. Unknown columns: [\"non-existent\"]"
-    ));
+    assert!(
+        result
+            .unwrap_err()
+            .to_string()
+            .contains("field \"non-existent\" not found in schema")
+    );
     Ok(())
 }
 
@@ -1499,6 +1502,121 @@ async fn test_zorder_respects_target_size() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+#[tokio::test]
+async fn test_zorder_nested_columns() -> Result<(), Box<dyn Error>> {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        Field::new(
+            "meta",
+            ArrowDataType::Struct(vec![Field::new("field_a", ArrowDataType::Int32, false)].into()),
+            false,
+        ),
+        Field::new("value", ArrowDataType::Int32, false),
+    ]));
+
+    let batch1 = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(arrow_array::StructArray::from(vec![(
+                Arc::new(Field::new("field_a", ArrowDataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![1, 2, 3])) as Arc<dyn arrow_array::Array>,
+            )])),
+            Arc::new(Int32Array::from(vec![10, 20, 30])),
+        ],
+    )?;
+
+    let batch2 = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(arrow_array::StructArray::from(vec![(
+                Arc::new(Field::new("field_a", ArrowDataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![4, 5, 6])) as Arc<dyn arrow_array::Array>,
+            )])),
+            Arc::new(Int32Array::from(vec![40, 50, 60])),
+        ],
+    )?;
+
+    let table = DeltaTable::new_in_memory()
+        .write(vec![batch1])
+        .with_save_mode(deltalake_core::protocol::SaveMode::Append)
+        .await?;
+
+    let table = table
+        .write(vec![batch2])
+        .with_save_mode(deltalake_core::protocol::SaveMode::Append)
+        .await?;
+
+    let (_, metrics) = table
+        .optimize()
+        .with_type(OptimizeType::ZOrder(vec!["meta.field_a".to_string()]))
+        .await?;
+
+    assert_eq!(metrics.num_files_added, 1);
+    assert_eq!(metrics.num_files_removed, 2);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_zorder_rejects_invalid_nested_path() -> Result<(), Box<dyn Error>> {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        Field::new(
+            "meta",
+            ArrowDataType::Struct(vec![Field::new("field_a", ArrowDataType::Int32, false)].into()),
+            false,
+        ),
+        Field::new("value", ArrowDataType::Int32, false),
+    ]));
+
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(arrow_array::StructArray::from(vec![(
+                Arc::new(Field::new("field_a", ArrowDataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![1, 2, 3])) as Arc<dyn arrow_array::Array>,
+            )])),
+            Arc::new(Int32Array::from(vec![10, 20, 30])),
+        ],
+    )?;
+
+    // Non-existent nested field
+    let table = DeltaTable::new_in_memory()
+        .write(vec![batch.clone()])
+        .with_save_mode(deltalake_core::protocol::SaveMode::Append)
+        .await?;
+
+    let result = table
+        .optimize()
+        .with_type(OptimizeType::ZOrder(vec!["meta.nonexistent".to_string()]))
+        .await;
+    assert!(result.is_err());
+    assert!(
+        result
+            .unwrap_err()
+            .to_string()
+            .contains("field \"nonexistent\" not found in schema")
+    );
+
+    // Non-struct intermediate field
+    let table = DeltaTable::new_in_memory()
+        .write(vec![batch])
+        .with_save_mode(deltalake_core::protocol::SaveMode::Append)
+        .await?;
+
+    let result = table
+        .optimize()
+        .with_type(OptimizeType::ZOrder(vec!["value.sub".to_string()]))
+        .await;
+    assert!(result.is_err());
+    assert!(
+        result
+            .unwrap_err()
+            .to_string()
+            .contains("\"value\" is not a struct type")
+    );
+
+    Ok(())
+}
+
 async fn read_parquet_file(
     path: &Path,
     object_store: ObjectStoreRef,