Repository: partiql/partiql-lang-rust
Pull request #559: Feat project all
Author: login: jpschorr
Created: 2025-04-10T22:49:43Z | Merged: 2025-04-11T19:15:34Z

Make project-all work with the modeling of `UNPIVOT`

---

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.



Diff:
diff --git a/extension/partiql-extension-visualize/src/plan_to_dot.rs b/extension/partiql-extension-visualize/src/plan_to_dot.rs
index 980e3d2c..bbf1b607 100644
--- a/extension/partiql-extension-visualize/src/plan_to_dot.rs
+++ b/extension/partiql-extension-visualize/src/plan_to_dot.rs
@@ -100,7 +100,7 @@ impl PlanToDot {
                         .join(" | "),
                 )
             }
-            BindingsOp::ProjectAll => "{project  * }".to_string(),
+            BindingsOp::ProjectAll(_) => "{project  * }".to_string(),
             BindingsOp::ProjectValue(pv) => {
                 format!("{{project value | {} }}", expr_to_str(&pv.expr))
             }
diff --git a/partiql-eval/benches/bench_eval.rs b/partiql-eval/benches/bench_eval.rs
index cf5eaabb..f9f17b5b 100644
--- a/partiql-eval/benches/bench_eval.rs
+++ b/partiql-eval/benches/bench_eval.rs
@@ -12,7 +12,8 @@ use partiql_eval::plan::EvaluationMode;
 use partiql_logical as logical;
 use partiql_logical::BindingsOp::{Project, ProjectAll};
 use partiql_logical::{
-    BinaryOp, BindingsOp, JoinKind, LogicalPlan, PathComponent, ValueExpr, VarRefType,
+    BinaryOp, BindingsOp, JoinKind, LogicalPlan, PathComponent, ProjectAllMode, ValueExpr,
+    VarRefType,
 };
 use partiql_value::{bag, list, tuple, BindingsName, DateTime, Value};
 
@@ -174,7 +175,7 @@ fn eval_bench(c: &mut Criterion) {
             as_key: "x".to_string(),
             at_key: None,
         }));
-        let project_all = logical_plan.add_operator(ProjectAll);
+        let project_all = logical_plan.add_operator(ProjectAll(ProjectAllMode::Unwrap));
         let sink = logical_plan.add_operator(BindingsOp::Sink);
 
         logical_plan.add_flow(from, project_all);
diff --git a/partiql-eval/src/eval/evaluable.rs b/partiql-eval/src/eval/evaluable.rs
index 6135b1d4..cd1cc826 100644
--- a/partiql-eval/src/eval/evaluable.rs
+++ b/partiql-eval/src/eval/evaluable.rs
@@ -1186,11 +1186,15 @@ impl Evaluable for EvalSelect {
 #[derive(Debug, Default)]
 pub(crate) struct EvalSelectAll {
     pub(crate) input: Option<Value>,
+    pub(crate) passthrough: bool,
 }
 
 impl EvalSelectAll {
-    pub(crate) fn new() -> Self {
-        Self::default()
+    pub(crate) fn new(passthrough: bool) -> Self {
+        Self {
+            passthrough,
+            ..Self::default()
+        }
     }
 }
 
@@ -1201,10 +1205,14 @@ impl Evaluable for EvalSelectAll {
         let ordered = input_value.is_ordered();
 
         let values = input_value.into_iter().map(|val| {
-            val.coerce_into_tuple()
-                .into_values()
-                .flat_map(|v| v.coerce_into_tuple().into_pairs())
-                .collect::<Tuple>()
+            if self.passthrough {
+                val.coerce_into_tuple()
+            } else {
+                val.coerce_into_tuple()
+                    .into_values()
+                    .flat_map(|v| v.coerce_into_tuple().into_pairs())
+                    .collect::<Tuple>()
+            }
         });
 
         match ordered {
diff --git a/partiql-eval/src/lib.rs b/partiql-eval/src/lib.rs
index d83857ad..3eb65421 100644
--- a/partiql-eval/src/lib.rs
+++ b/partiql-eval/src/lib.rs
@@ -1478,7 +1478,7 @@ mod tests {
 
         let from = lg.add_operator(scan("data", "data"));
 
-        let project = lg.add_operator(ProjectAll);
+        let project = lg.add_operator(ProjectAll(Default::default()));
 
         let sink = lg.add_operator(BindingsOp::Sink);
 
diff --git a/partiql-eval/src/plan.rs b/partiql-eval/src/plan.rs
index 9e0fd870..71b1981e 100644
--- a/partiql-eval/src/plan.rs
+++ b/partiql-eval/src/plan.rs
@@ -21,8 +21,8 @@ use partiql_logical as logical;
 use partiql_logical::{
     AggFunc, BagOperator, BinaryOp, BindingsOp, CallName, GraphMatchExpr, GroupingStrategy,
     IsTypeExpr, JoinKind, Lit, LogicalPlan, OpId, PathComponent, Pattern, PatternMatchExpr,
-    SearchedCase, SetQuantifier, SortSpecNullOrder, SortSpecOrder, Type, UnaryOp, ValueExpr,
-    VarRefType,
+    ProjectAllMode, SearchedCase, SetQuantifier, SortSpecNullOrder, SortSpecOrder, Type, UnaryOp,
+    ValueExpr, VarRefType,
 };
 use partiql_value::boxed_variant::DynBoxedVariantTypeFactory;
 use partiql_value::{Bag, List, Tuple, Value, Variant};
@@ -198,7 +198,9 @@ impl<'c> EvaluatorPlanner<'c> {
                     .collect();
                 Box::new(eval::evaluable::EvalSelect::new(exprs))
             }
-            BindingsOp::ProjectAll => Box::new(eval::evaluable::EvalSelectAll::new()),
+            BindingsOp::ProjectAll(mode) => Box::new(eval::evaluable::EvalSelectAll::new(
+                mode == &ProjectAllMode::PassThrough,
+            )),
             BindingsOp::ProjectValue(logical::ProjectValue { expr }) => {
                 let expr = self.plan_value::<{ STRICT }>(expr);
                 Box::new(eval::evaluable::EvalSelectValue::new(expr))
diff --git a/partiql-logical-planner/src/lower.rs b/partiql-logical-planner/src/lower.rs
index 5baf792d..d739cc63 100644
--- a/partiql-logical-planner/src/lower.rs
+++ b/partiql-logical-planner/src/lower.rs
@@ -17,7 +17,7 @@ use partiql_logical as logical;
 use partiql_logical::{
     AggregateExpression, BagExpr, BagOp, BetweenExpr, BindingsOp, GraphMatchExpr, IsTypeExpr,
     LikeMatch, LikeNonStringNonLiteralMatch, ListExpr, LogicalPlan, OpId, PathComponent, Pattern,
-    PatternMatchExpr, SortSpecOrder, TupleExpr, ValueExpr, VarRefType,
+    PatternMatchExpr, ProjectAllMode, SortSpecOrder, TupleExpr, ValueExpr, VarRefType,
 };
 use std::borrow::Cow;
 
@@ -879,7 +879,7 @@ impl<'ast> Visitor<'ast> for AstToLogical<'_> {
         let env = self.exit_env();
 
         let select: BindingsOp = match _projection_kind {
-            ProjectionKind::ProjectStar => logical::BindingsOp::ProjectAll,
+            ProjectionKind::ProjectStar => logical::BindingsOp::ProjectAll(Default::default()),
             ProjectionKind::ProjectList(_) => {
                 true_or_fault!(self, env.len().is_even(), "env.len() is not even");
                 let mut exprs = Vec::with_capacity(env.len() / 2);
@@ -1501,25 +1501,40 @@ impl<'ast> Visitor<'ast> for AstToLogical<'_> {
             .as_ref()
             .map(|SymbolPrimitive { value, case: _ }| value.clone());
 
-        let bexpr = match kind {
-            FromLetKind::Scan => logical::BindingsOp::Scan(logical::Scan {
-                expr,
-                as_key,
-                at_key,
-            }),
-            FromLetKind::Unpivot => logical::BindingsOp::Unpivot(logical::Unpivot {
-                expr,
-                as_key,
-                at_key,
-            }),
-            FromLetKind::GraphTable => logical::BindingsOp::Scan(logical::Scan {
-                expr,
-                as_key,
-                at_key,
-            }),
+        let (bexpr, project_all_mode) = match kind {
+            FromLetKind::Scan => (
+                logical::BindingsOp::Scan(logical::Scan {
+                    expr,
+                    as_key,
+                    at_key,
+                }),
+                ProjectAllMode::Unwrap,
+            ),
+            FromLetKind::Unpivot => (
+                logical::BindingsOp::Unpivot(logical::Unpivot {
+                    expr,
+                    as_key,
+                    at_key,
+                }),
+                ProjectAllMode::PassThrough,
+            ),
+            FromLetKind::GraphTable => (
+                logical::BindingsOp::Scan(logical::Scan {
+                    expr,
+                    as_key,
+                    at_key,
+                }),
+                ProjectAllMode::Unwrap,
+            ),
         };
         let id = self.plan.add_operator(bexpr);
         self.push_bexpr(id);
+        if let Some(select_id) = self.current_clauses_mut().select_clause {
+            if let Some(BindingsOp::ProjectAll(mode)) = self.plan.operator_as_mut(select_id) {
+                *mode = project_all_mode
+            }
+        }
+
         Traverse::Continue
     }
 
@@ -1671,7 +1686,7 @@ impl<'ast> Visitor<'ast> for AstToLogical<'_> {
         let mut binding = Vec::new();
         let select_clause_exprs = match select_clause {
             BindingsOp::Project(ref mut project) => &mut project.exprs,
-            BindingsOp::ProjectAll => &mut binding,
+            BindingsOp::ProjectAll(_) => &mut binding,
             BindingsOp::ProjectValue(_) => &mut binding, // TODO: replacement of SELECT VALUE expressions
             _ => {
                 self.errors.push(AstTransformError::IllegalState(
diff --git a/partiql-logical/src/lib.rs b/partiql-logical/src/lib.rs
index 84972f24..e8d5884d 100644
--- a/partiql-logical/src/lib.rs
+++ b/partiql-logical/src/lib.rs
@@ -218,7 +218,7 @@ pub enum BindingsOp {
     Join(Join),
     BagOp(BagOp),
     Project(Project),
-    ProjectAll,
+    ProjectAll(ProjectAllMode),
     ProjectValue(ProjectValue),
     ExprQuery(ExprQuery),
     Distinct,
@@ -228,6 +228,14 @@ pub enum BindingsOp {
     Sink,
 }
 
+#[derive(Debug, Clone, Default, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum ProjectAllMode {
+    #[default]
+    Unwrap,
+    PassThrough,
+}
+
 /// [`Scan`] bridges from [`ValueExpr`]s to [`BindingsOp`]s.
 #[derive(Debug, Clone, Eq, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
diff --git a/partiql-value/src/value.rs b/partiql-value/src/value.rs
index ec23735c..7a5efc88 100644
--- a/partiql-value/src/value.rs
+++ b/partiql-value/src/value.rs
@@ -17,7 +17,7 @@ mod math;
 
 use crate::datum::{
     Datum, DatumCategory, DatumCategoryOwned, DatumCategoryRef, DatumLower, DatumLowerResult,
-    DatumSeqRef, DatumTupleOwned, DatumTupleRef, DatumValue,
+    DatumTupleOwned, DatumTupleRef, DatumValue,
 };
 pub use iter::*;
 pub use logic::*;
diff --git a/partiql/tests/queries.rs b/partiql/tests/queries.rs
new file mode 100644
index 00000000..28ff29be
--- /dev/null
+++ b/partiql/tests/queries.rs
@@ -0,0 +1,30 @@
+use crate::common::{eval_query_with_catalog, TestError};
+use assert_matches::assert_matches;
+use partiql_catalog::catalog::PartiqlCatalog;
+use partiql_catalog::extension::Extension;
+use partiql_eval::eval::Evaluated;
+use partiql_eval::plan::EvaluationMode;
+use partiql_extension_value_functions::PartiqlValueFnExtension;
+use partiql_value::Value;
+
+mod common;
+
+#[track_caller]
+#[inline]
+pub fn eval(statement: &str, mode: EvaluationMode) -> Result<Evaluated, TestError<'_>> {
+    let mut catalog = PartiqlCatalog::default();
+    let ext = PartiqlValueFnExtension::default();
+    ext.load(&mut catalog)?;
+
+    eval_query_with_catalog(statement, &catalog, mode)
+}
+
+#[test]
+fn select_star_unpivot() {
+    let query = "SELECT * FROM UNPIVOT {'amzn': 840.05, 'tdc': 31.06} AS price AT sym";
+
+    let res = eval(query, EvaluationMode::Permissive);
+    assert_matches!(res, Ok(_));
+    let res = res.unwrap().result;
+    insta::assert_debug_snapshot!(res);
+}
diff --git a/partiql/tests/snapshots/queries__select_star_unpivot.snap b/partiql/tests/snapshots/queries__select_star_unpivot.snap
new file mode 100644
index 00000000..6b2d3458
--- /dev/null
+++ b/partiql/tests/snapshots/queries__select_star_unpivot.snap
@@ -0,0 +1,5 @@
+---
+source: partiql/tests/queries.rs
+expression: res
+---
+<<{ 'price': 31.06, 'sym': 'tdc' }, { 'price': 840.05, 'sym': 'amzn' }>>


Reviews:
user: login: alancai98
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T19:07:08Z
user: login: jpschorr
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T19:16:55Z
user: login: jpschorr
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T19:17:01Z

Review comments:
diff_hunk: @@ -0,0 +1,30 @@
+use crate::common::{eval_query_with_catalog, TestError};
+use assert_matches::assert_matches;
+use partiql_catalog::catalog::PartiqlCatalog;
+use partiql_catalog::extension::Extension;
+use partiql_eval::eval::Evaluated;
+use partiql_eval::plan::EvaluationMode;
+use partiql_extension_value_functions::PartiqlValueFnExtension;
+use partiql_value::Value;
+
+mod common;
+
+#[track_caller]
+#[inline]
+pub fn eval(statement: &str, mode: EvaluationMode) -> Result<Evaluated, TestError<'_>> {
+    let mut catalog = PartiqlCatalog::default();
+    let ext = PartiqlValueFnExtension::default();
+    ext.load(&mut catalog)?;
+
+    eval_query_with_catalog(statement, &catalog, mode)
+}
+
+#[test]
+fn select_star_unpivot() {
+    let query = "SELECT * FROM UNPIVOT {'amzn': 840.05, 'tdc': 31.06} AS price AT sym";
path: partiql/tests/queries.rs
user: login: alancai98
body: could be a good test we port to conformance tests
created_at: 2025-04-11T19:06:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -228,6 +228,14 @@ pub enum BindingsOp {
     Sink,
 }
 
+#[derive(Debug, Clone, Default, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum ProjectAllMode {
+    #[default]
+    Unwrap,
+    PassThrough,
path: partiql-logical/src/lib.rs
user: login: alancai98
body: nit: could add some docs for what these variants represent
created_at: 2025-04-11T19:07:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,30 @@
+use crate::common::{eval_query_with_catalog, TestError};
+use assert_matches::assert_matches;
+use partiql_catalog::catalog::PartiqlCatalog;
+use partiql_catalog::extension::Extension;
+use partiql_eval::eval::Evaluated;
+use partiql_eval::plan::EvaluationMode;
+use partiql_extension_value_functions::PartiqlValueFnExtension;
+use partiql_value::Value;
+
+mod common;
+
+#[track_caller]
+#[inline]
+pub fn eval(statement: &str, mode: EvaluationMode) -> Result<Evaluated, TestError<'_>> {
+    let mut catalog = PartiqlCatalog::default();
+    let ext = PartiqlValueFnExtension::default();
+    ext.load(&mut catalog)?;
+
+    eval_query_with_catalog(statement, &catalog, mode)
+}
+
+#[test]
+fn select_star_unpivot() {
+    let query = "SELECT * FROM UNPIVOT {'amzn': 840.05, 'tdc': 31.06} AS price AT sym";
path: partiql/tests/queries.rs
user: login: jpschorr
body: https://github.com/partiql/partiql-tests/issues/134
created_at: 2025-04-11T19:16:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -228,6 +228,14 @@ pub enum BindingsOp {
     Sink,
 }
 
+#[derive(Debug, Clone, Default, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum ProjectAllMode {
+    #[default]
+    Unwrap,
+    PassThrough,
path: partiql-logical/src/lib.rs
user: login: jpschorr
body: will do in later PR
created_at: 2025-04-11T19:17:01Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-10T22:57:03Z
body: ### Conformance comparison report
| | Base (f998a5544c6637320bf835b75d5cc181616ae83a) | d5f67aed9a002768d1a679d80fdb4a66e7bb09b7 | +/- |
| --- | ---: | ---: | ---: |
| % Passing | 87.54% | 87.54% | 0.00% |
| :white_check_mark: Passing | 5777 | 5777 | 0 |
| :x: Failing | 822 | 822 | 0 |
| :large_orange_diamond: Ignored | 0 | 0 | 0 |
| Total Tests | 6599 | 6599 | 0 |

Number passing in both: 5777

Number failing in both: 822

Number passing in Base (f998a5544c6637320bf835b75d5cc181616ae83a) but now fail: 0

Number failing in Base (f998a5544c6637320bf835b75d5cc181616ae83a) but now pass: 0

author_association: NONE
user: login: codecov[bot]
created_at: 2025-04-10T22:57:04Z
body: ## [Codecov](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql) Report
Attention: Patch coverage is `93.75000%` with `3 lines` in your changes missing coverage. Please review.
> Project coverage is 82.06%. Comparing base [(`f998a55`)](https://app.codecov.io/gh/partiql/partiql-lang-rust/commit/f998a5544c6637320bf835b75d5cc181616ae83a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql) to head [(`cda7037`)](https://app.codecov.io/gh/partiql/partiql-lang-rust/commit/cda7037d2727c8876591a4a72719fbb5af0f13e1?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql).
> Report is 1 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql) | Patch % | Lines |
|---|---|---|
| [partiql-logical-planner/src/lower.rs](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?src=pr&el=tree&filepath=partiql-logical-planner%2Fsrc%2Flower.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql#diff-cGFydGlxbC1sb2dpY2FsLXBsYW5uZXIvc3JjL2xvd2VyLnJz) | 93.75% | [2 Missing :warning: ](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql) |
| [...ion/partiql-extension-visualize/src/plan\_to\_dot.rs](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?src=pr&el=tree&filepath=extension%2Fpartiql-extension-visualize%2Fsrc%2Fplan_to_dot.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql#diff-ZXh0ZW5zaW9uL3BhcnRpcWwtZXh0ZW5zaW9uLXZpc3VhbGl6ZS9zcmMvcGxhbl90b19kb3QucnM=) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #559      +/-   ##
==========================================
+ Coverage   82.05%   82.06%   +0.01%     
==========================================
  Files         110      110              
  Lines       23243    23264      +21     
  Branches    23243    23264      +21     
==========================================
+ Hits        19072    19092      +20     
- Misses       3633     3634       +1     
  Partials      538      538              
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/partiql/partiql-lang-rust/pull/559?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=partiql).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE