Repository: prql/prql
Pull request #429 (2022-04-27 13:35:18)

Refactor semantic module & materializer


Patch:
diff --git a/prql-compiler/src/ast/ast_fold.rs b/prql-compiler/src/ast/ast_fold.rs
index 180ab26e..6ecc2962 100644
--- a/prql-compiler/src/ast/ast_fold.rs
+++ b/prql-compiler/src/ast/ast_fold.rs
@@ -32,6 +32,7 @@ pub trait AstFold {
     }
     fn fold_table(&mut self, table: Table) -> Result<Table> {
         Ok(Table {
+            id: table.id,
             name: self.fold_ident(table.name)?,
             pipeline: Box::new(self.fold_node(*table.pipeline)?),
         })
@@ -84,12 +85,7 @@ pub fn fold_item<T: ?Sized + AstFold>(fold: &mut T, item: Item) -> Result<Item>
     Ok(match item {
         Item::Ident(ident) => Item::Ident(fold.fold_ident(ident)?),
         Item::Expr(items) => Item::Expr(fold.fold_nodes(items)?),
-        Item::List(items) => Item::List(
-            items
-                .into_iter()
-                .map(|x| fold.fold_node(x.into_inner()).map(ListItem))
-                .try_collect()?,
-        ),
+        Item::List(items) => Item::List(fold.fold_nodes(items)?),
         Item::Range(Range { start, end }) => Item::Range(Range {
             start: fold_optional_box(fold, start)?,
             end: fold_optional_box(fold, end)?,
@@ -233,6 +229,7 @@ pub fn fold_table_ref<T: ?Sized + AstFold>(fold: &mut T, table: TableRef) -> Res
     Ok(TableRef {
         name: fold.fold_ident(table.name)?,
         alias: table.alias.map(|a| fold.fold_ident(a)).transpose()?,
+        ..table
     })
 }
 
diff --git a/prql-compiler/src/ast/item.rs b/prql-compiler/src/ast/item.rs
index 2fd4d96d..f05c47a1 100644
--- a/prql-compiler/src/ast/item.rs
+++ b/prql-compiler/src/ast/item.rs
@@ -19,7 +19,7 @@ pub enum Item {
     Query(Query),
     Pipeline(Pipeline),
     Transform(Transform),
-    List(Vec<ListItem>),
+    List(Vec<Node>),
     Range(Range),
     Expr(Vec<Node>),
     FuncDef(FuncDef),
@@ -33,12 +33,6 @@ pub enum Item {
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub struct ListItem(pub Node);
 
-impl ListItem {
-    pub fn into_inner(self) -> Node {
-        self.0
-    }
-}
-
 /// Function call.
 ///
 /// Note that `named_args` cannot be determined during parsing, but only during resolving.
@@ -181,11 +175,11 @@ impl Display for Item {
                 if nodes.is_empty() {
                     f.write_str("[]")?;
                 } else if nodes.len() == 1 {
-                    write!(f, "[{}]", nodes[0].0.item)?;
+                    write!(f, "[{}]", nodes[0].item)?;
                 } else {
                     f.write_str("[\n")?;
                     for li in nodes.iter() {
-                        writeln!(f, "  {},", li.0.item)?;
+                        writeln!(f, "  {},", li.item)?;
                     }
                     f.write_str("]")?;
                 }
diff --git a/prql-compiler/src/ast/mod.rs b/prql-compiler/src/ast/mod.rs
index fd5a492b..67a3c8f9 100644
--- a/prql-compiler/src/ast/mod.rs
+++ b/prql-compiler/src/ast/mod.rs
@@ -1,14 +1,13 @@
 /// Abstract syntax tree for PRQL language
 ///
 /// The central struct here is [Node], that can be of different kinds, described with [item::Item].
-use anyhow::{bail, Result};
+use anyhow::Result;
 use serde::{Deserialize, Serialize};
 
 pub use self::dialect::*;
 pub use self::item::*;
 pub use self::query::*;
 use crate::error::{Error, Reason, Span};
-use crate::semantic::Frame;
 use crate::utils::*;
 
 pub mod ast_fold;
@@ -31,28 +30,13 @@ pub struct Node {
     pub span: Option<Span>,
     #[serde(skip)]
     pub declared_at: Option<usize>,
-    #[serde(skip)]
-    pub frame: Option<Frame>,
 }
 
 impl Node {
-    /// For lists that only have one item in each ListItem this returns a Vec of
-    /// those terms. (e.g. `[1, a b]` but not `[1 + 2]`, because `+` in an
-    /// operator and so will create an `Items` for each of `1` & `2`)
-    pub fn into_inner_list_nodes(self) -> Result<Vec<Node>> {
-        Ok(match self.item {
-            Item::List(items) => items.into_iter().map(|x| x.into_inner()).collect(),
-            _ => bail!("Expected a list of single items, got {self:?}"),
-        })
-    }
-
-    /// Make a List from a vec of Items
-    pub fn into_list_of_nodes(node: Vec<Node>) -> Node {
-        Item::List(node.into_iter().map(ListItem).collect()).into()
-    }
-
     /// Return an error if this is named expression.
     pub fn discard_name(self) -> Result<Node, Error> {
+        // TODO: replace this function with a prior type checking
+
         if let Item::NamedExpr(_) = self.item {
             Err(Error::new(Reason::Unexpected {
                 found: "alias".to_string(),
@@ -64,26 +48,19 @@ impl Node {
     }
 
     pub fn into_name_and_expr(self) -> (Option<Ident>, Node) {
-        // unwrap expr with only one child
-        let expr = if let Item::Expr(mut expr) = self.item {
-            expr.remove(0)
-        } else {
-            self
-        };
-
-        if let Item::NamedExpr(expr) = expr.item {
+        if let Item::NamedExpr(expr) = self.item {
             (Some(expr.name), *expr.expr)
         } else {
-            (None, expr)
+            (None, self)
         }
     }
 
     /// Often we don't care whether a List or single item is passed; e.g.
     /// `select x` vs `select [x, y]`. This equalizes them both to a vec of
-    /// Item-ss.
-    pub fn coerce_to_items(self) -> Vec<Node> {
+    /// [Node]-s.
+    pub fn coerce_to_vec(self) -> Vec<Node> {
         match self.item {
-            Item::List(items) => items.into_iter().map(|x| x.into_inner()).collect(),
+            Item::List(items) => items,
             _ => vec![self],
         }
     }
@@ -103,7 +80,7 @@ impl Node {
     }
 }
 
-/// Unnest Expr([x]) into x.
+/// Wrap into [Item::Expr] if there is more than one term.
 pub trait IntoExpr {
     fn into_expr(self) -> Item;
 }
@@ -123,7 +100,6 @@ impl From<Item> for Node {
             item,
             span: None,
             declared_at: None,
-            frame: None,
         }
     }
 }
diff --git a/prql-compiler/src/ast/query.rs b/prql-compiler/src/ast/query.rs
index e52c4866..1d027e95 100644
--- a/prql-compiler/src/ast/query.rs
+++ b/prql-compiler/src/ast/query.rs
@@ -37,6 +37,7 @@ pub enum FuncKind {
 pub struct Table {
     pub name: String,
     pub pipeline: Box<Node>,
+    pub id: Option<usize>,
 }
 
 /// Transform is a stage of a pipeline. It is created from a FuncCall during parsing.
@@ -82,6 +83,7 @@ impl Select {
 pub struct TableRef {
     pub name: String,
     pub alias: Option<String>,
+    pub declared_at: Option<usize>,
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
diff --git a/prql-compiler/src/cli.rs b/prql-compiler/src/cli.rs
index 67fe4334..92d3ed80 100644
--- a/prql-compiler/src/cli.rs
+++ b/prql-compiler/src/cli.rs
@@ -114,7 +114,8 @@ fn compile_to(format: Format, source: &str) -> Result<Vec<u8>, Error> {
             let (nodes, context) = semantic::resolve(query.nodes, Some(context))?;
 
             semantic::label_references(&nodes, &context, "".to_string(), source.to_string());
-            vec![]
+
+            format!("{context:?}").as_bytes().to_vec()
         }
         Format::PrqlFrames => {
             let query = parse(source)?;
diff --git a/prql-compiler/src/parser.rs b/prql-compiler/src/parser.rs
index 446bd1eb..6caeafbb 100644
--- a/prql-compiler/src/parser.rs
+++ b/prql-compiler/src/parser.rs
@@ -94,12 +94,7 @@ fn ast_of_parse_tree(pairs: Pairs<Rule>) -> Result<Vec<Node>> {
                         dialect,
                     })
                 }
-                Rule::list => Item::List(
-                    ast_of_parse_tree(pair.into_inner())?
-                        .into_iter()
-                        .map(ListItem)
-                        .collect(),
-                ),
+                Rule::list => Item::List(ast_of_parse_tree(pair.into_inner())?),
                 Rule::expr_mul
                 | Rule::expr_add
                 | Rule::expr_compare
@@ -169,6 +164,7 @@ fn ast_of_parse_tree(pairs: Pairs<Rule>) -> Result<Vec<Node>> {
                         .try_into()
                         .map_err(|e| anyhow!("Expected two items; {e:?}"))?;
                     Item::Table(Table {
+                        id: None,
                         name: name.item.into_ident()?,
                         pipeline: Box::new(pipeline),
                     })
@@ -923,6 +919,7 @@ take 20
                     args:
                       - Ident: employees
                     named_args: {}
+          id: ~
         "###);
 
         assert_yaml_snapshot!(ast_of_string(
@@ -981,6 +978,7 @@ take 20
                     args:
                       - Raw: "50"
                     named_args: {}
+          id: ~
         "###);
         Ok(())
     }
diff --git a/prql-compiler/src/prql.pest b/prql-compiler/src/prql.pest
index e96d1201..09c25bbf 100644
--- a/prql-compiler/src/prql.pest
+++ b/prql-compiler/src/prql.pest
@@ -26,7 +26,7 @@ table = { "table" ~ ident ~ "=" ~ nested_pipeline }
 pipe = _{ NEWLINE | "|" }
 pipeline = { WHITESPACE* ~ func_curry ~ (pipe ~ func_curry)* }
 
-ident = @{ !operator ~ !keyword ~ (ASCII_ALPHA | "$") ~ (ASCII_ALPHANUMERIC | "." | "_" | "*" )* }
+ident = @{ !operator ~ !(keyword ~ WHITESPACE) ~ (ASCII_ALPHA | "$") ~ (ASCII_ALPHANUMERIC | "." | "_" | "*" )* }
 keyword = _{ "prql" | "table" | "func" }
 
 // A central issue around the terms vs expr is that we want to be able to parse:
diff --git a/prql-compiler/src/semantic/context.rs b/prql-compiler/src/semantic/context.rs
index 70f7b7bd..2a27739f 100644
--- a/prql-compiler/src/semantic/context.rs
+++ b/prql-compiler/src/semantic/context.rs
@@ -1,50 +1,24 @@
 use anyhow::Result;
 use enum_as_inner::EnumAsInner;
-use serde::Deserialize;
-use serde::Serialize;
-use std::collections::HashMap;
+use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
+use std::fmt::Debug;
 
+use super::scope::NS_PARAM;
+use super::{split_var_name, Scope};
 use crate::ast::*;
 use crate::error::Span;
 
-const NS_PARAM: &str = "_param";
-const NS_GLOB: &str = "_glob";
-
-/// Scope within which we can reference variables, functions and tables
-/// Provides fast lookups for different names.
-#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+/// Context of the pipeline.
+#[derive(Default, Serialize, Deserialize, Clone)]
 pub struct Context {
-    /// current table columns (result of last pipeline)
-    pub frame: Frame,
-
-    /// Mapping from idents to their declarations. For each namespace (table), a map from column names to their definitions
-    /// "_param" is namespace of current function parameters
-    /// "_glob" is namespace of functions without parameters (global variables)
-    pub(crate) variables: HashMap<String, HashSet<usize>>,
-
-    pub(crate) functions: HashMap<String, usize>,
+    /// Map of all accessible names (for each namespace)
+    pub(crate) scope: Scope,
 
     /// All declarations, even those out of scope
     pub(crate) declarations: Vec<(Declaration, Option<Span>)>,
 }
 
-#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub struct Frame {
-    pub columns: Vec<TableColumn>,
-    pub sort: Vec<ColumnSort<usize>>,
-    pub group: Vec<usize>,
-
-    pub tables: Vec<usize>,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum TableColumn {
-    All(usize),
-    Unnamed(usize),
-    Named(String, usize),
-}
-
 #[derive(Debug, EnumAsInner, Clone, Serialize, Deserialize, strum::Display)]
 pub enum Declaration {
     Expression(Box<Node>),
@@ -59,110 +33,7 @@ pub enum Declaration {
     Function(FuncDef),
 }
 
-impl Frame {
-    pub fn groups_to_columns(&mut self) {
-        for col in &self.group {
-            self.columns.push(TableColumn::Unnamed(*col))
-        }
-    }
-
-    pub fn decls_in_use(&self) -> HashSet<usize> {
-        let mut r = HashSet::new();
-        for col in &self.columns {
-            match col {
-                TableColumn::Unnamed(id) | TableColumn::Named(_, id) => {
-                    r.insert(*id);
-                }
-                _ => {}
-            }
-        }
-        for col in &self.group {
-            r.insert(*col);
-        }
-        r
-    }
-
-    pub fn get_column_names(&self, context: &Context) -> Vec<Option<String>> {
-        self.columns
-            .iter()
-            .map(|col| match col {
-                TableColumn::All(namespace) => {
-                    let (table, _) = &context.declarations[*namespace];
-                    let table = table.as_table().map(|x| x.as_str()).unwrap_or("");
-                    Some(format!("{table}.*"))
-                }
-                TableColumn::Unnamed(_) => None,
-                TableColumn::Named(name, _) => Some(name.clone()),
-            })
-            .collect()
-    }
-}
-
 impl Context {
-    pub(crate) fn replace_declaration(&mut self, id: usize, new_decl: Declaration) {
-        let (decl, _) = self.declarations.get_mut(id).unwrap();
-        *decl = new_decl;
-    }
-
-    pub(crate) fn replace_declaration_expr(&mut self, id: usize, expr: Node) {
-        self.replace_declaration(id, Declaration::Expression(Box::new(expr)));
-    }
-
-    /// Removes all names from scopes, except functions and columns in frame.
-    pub(super) fn clear_scope(&mut self) {
-        let in_use = self.frame.decls_in_use();
-
-        self.variables.retain(|name, decls| {
-            if let Some(id) = decls.iter().find(|id| in_use.contains(id)).cloned() {
-                decls.clear();
-                decls.insert(id);
-                true
-            } else {
-                name.starts_with(NS_GLOB)
-            }
-        });
-
-        let to_cascade: Vec<_> = self.variables.keys().cloned().collect();
-
-        for name in to_cascade {
-            self.cascade_variable(name.as_str());
-        }
-    }
-
-    pub fn print(&self) {
-        for (i, (d, _)) in self.declarations.iter().enumerate() {
-            match d {
-                Declaration::Expression(v) => {
-                    println!("[{i:3}]: expr  `{}`", v.item);
-                }
-                Declaration::ExternRef { table, variable } => {
-                    println!("[{i:3}]: col   `{variable}` from table {table:?}");
-                }
-                Declaration::Table(name) => {
-                    println!("[{i:3}]: table `{name}`");
-                }
-                Declaration::Function(f) => {
-                    println!("[{i:3}]: func  `{}`", f.name);
-                }
-            }
-        }
-        print!("[");
-        for t_col in &self.frame.columns {
-            match t_col {
-                TableColumn::All(ns) => {
-                    print!(" {ns}.* ")
-                }
-                TableColumn::Named(name, id) => {
-                    print!(" {name}:{id} ")
-                }
-                TableColumn::Unnamed(id) => {
-                    print!(" {id} ")
-                }
-            }
-        }
-        println!("]");
-    }
-
     pub fn declare(&mut self, dec: Declaration, span: Option<Span>) -> usize {
         self.declarations.push((dec, span));
         self.declarations.len() - 1
@@ -170,33 +41,26 @@ impl Context {
 
     pub fn declare_func(&mut self, func_def: FuncDef) -> usize {
         let name = func_def.name.clone();
-        let no_params = func_def.named_params.is_empty() && func_def.positional_params.is_empty();
+        let has_params =
+            !func_def.named_params.is_empty() || !func_def.positional_params.is_empty();
 
         let span = func_def.body.span;
         let id = self.declare(Declaration::Function(func_def), span);
 
-        if no_params {
-            let name = format!("{NS_GLOB}.{name}");
-            self.add_to_scope(&name, id);
-        } else {
-            self.functions.insert(name, id);
-        }
+        self.scope.add_function(name, id, has_params);
 
         id
     }
 
-    pub fn declare_table(&mut self, t: &TableRef) {
+    pub fn declare_table(&mut self, t: &mut TableRef) {
         let name = t.alias.clone().unwrap_or_else(|| t.name.clone());
         let decl = Declaration::Table(name.clone());
 
         let table_id = self.declare(decl, None);
-        self.frame.tables.push(table_id);
+        t.declared_at = Some(table_id);
 
         let var_name = format!("{name}.*");
-        self.add_to_scope(var_name.as_str(), table_id);
-
-        let column = TableColumn::All(table_id);
-        self.frame.columns.push(column);
+        self.scope.add(var_name, table_id);
     }
 
     pub fn declare_func_param(&mut self, node: &Node) -> usize {
@@ -209,42 +73,17 @@ impl Context {
         // doesn't matter, will get overridden anyway
         let decl = Box::new(Item::Ident("".to_string()).into());
 
-        let name = format!("{NS_PARAM}.{name}");
         let id = self.declare(Declaration::Expression(decl), None);
 
-        self.add_to_scope(&name, id);
+        self.scope.add(format!("{NS_PARAM}.{name}"), id);
 
         id
     }
 
-    pub fn add_to_scope(&mut self, ident: &str, id: usize) {
-        // insert into own namespace, override other declarations
-        let decls = self.variables.entry(ident.to_string()).or_default();
-        let overridden = decls.drain().next();
-        decls.insert(id);
-
-        // remove overridden columns from frame
-        if let Some(overridden) = overridden {
-            self.frame.columns.retain(|col| col != &overridden);
-        }
-
-        self.cascade_variable(ident);
-    }
-
-    // insert into lower namespaces, possibly creating ambiguities
-    fn cascade_variable(&mut self, ident: &str) {
-        let id = *self.variables[ident].iter().next().unwrap();
-
-        let (_, var_name) = split_var_name(ident);
-
-        let decls = self.variables.entry(var_name.to_string()).or_default();
-        decls.insert(id);
-    }
-
     pub fn lookup_variable(&mut self, ident: &str, span: Option<Span>) -> Result<usize, String> {
         let (namespace, variable) = split_var_name(ident);
 
-        if let Some(decls) = self.variables.get(ident) {
+        if let Some(decls) = self.scope.variables.get(ident) {
             // lookup the inverse index
 
             match decls.len() {
@@ -266,7 +105,7 @@ impl Context {
                 format!("{namespace}.*")
             };
 
-            if let Some(decls) = self.variables.get(&all) {
+            if let Some(decls) = self.scope.variables.get(&all) {
                 // this variable can be from a namespace that we don't know all columns of
 
                 match decls.len() {
@@ -281,7 +120,7 @@ impl Context {
                             variable: variable.to_string(),
                         };
                         let id = self.declare(decl, span);
-                        self.add_to_scope(ident, id);
+                        self.scope.add(ident.to_string(), id);
 
                         Ok(id)
                     }
@@ -305,21 +144,16 @@ impl Context {
 
     pub fn lookup_namespaces_of(&mut self, variable: &str) -> HashSet<usize> {
         let mut r = HashSet::new();
-        if let Some(ns) = self.variables.get(variable) {
+        if let Some(ns) = self.scope.variables.get(variable) {
             r.extend(ns.clone());
         }
-        if let Some(ns) = self.variables.get("*") {
+        if let Some(ns) = self.scope.variables.get("*") {
             r.extend(ns.clone());
         }
         r
     }
 }
 
-/// Splits ident into namespaces and variable name
-pub fn split_var_name(ident: &str) -> (&str, &str) {
-    ident.rsplit_once('.').unwrap_or(("", ident))
-}
-
 impl From<Declaration> for anyhow::Error {
     fn from(dec: Declaration) -> Self {
         // panic!("Unexpected declaration type: {dec:?}");
@@ -327,11 +161,24 @@ impl From<Declaration> for anyhow::Error {
     }
 }
 
-impl PartialEq<usize> for TableColumn {
-    fn eq(&self, other: &usize) -> bool {
-        match self {
-            TableColumn::All(_) => false,
-            TableColumn::Unnamed(id) | TableColumn::Named(_, id) => id == other,
+impl Debug for Context {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for (i, (d, _)) in self.declarations.iter().enumerate() {
+            match d {
+                Declaration::Expression(v) => {
+                    writeln!(f, "[{i:3}]: expr  `{}`", v.item)?;
+                }
+                Declaration::ExternRef { table, variable } => {
+                    writeln!(f, "[{i:3}]: col   `{variable}` from table {table:?}")?;
+                }
+                Declaration::Table(name) => {
+                    writeln!(f, "[{i:3}]: table `{name}`")?;
+                }
+                Declaration::Function(func) => {
+                    writeln!(f, "[{i:3}]: func  `{}`", func.name)?;
+                }
+            }
         }
+        Ok(())
     }
 }
diff --git a/prql-compiler/src/semantic/frame.rs b/prql-compiler/src/semantic/frame.rs
new file mode 100644
index 00000000..2aee4f85
--- /dev/null
+++ b/prql-compiler/src/semantic/frame.rs
@@ -0,0 +1,172 @@
+use std::fmt::{Debug, Formatter};
+
+use anyhow::{anyhow, Result};
+use itertools::Itertools;
+use serde::{Deserialize, Serialize};
+
+use super::Context;
+use crate::ast::{ColumnSort, Item, Node, Transform};
+
+/// Represents the object that is manipulated by the pipeline transforms.
+/// Similar to a view in a database or a data frame.
+#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
+pub struct Frame {
+    pub columns: Vec<FrameColumn>,
+    pub sort: Vec<ColumnSort<usize>>,
+
+    pub tables: Vec<usize>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum FrameColumn {
+    All(usize),
+    Unnamed(usize),
+    Named(String, usize),
+}
+
+impl Frame {
+    fn push_column(&mut self, name: Option<String>, id: usize) {
+        // remove columns with the same name
+        if let Some(name) = &name {
+            self.columns.retain(|c| match c {
+                FrameColumn::Named(n, _) => n != name,
+                _ => true,
+            })
+        }
+
+        let column = if let Some(name) = name {
+            FrameColumn::Named(name, id)
+        } else {
+            FrameColumn::Unnamed(id)
+        };
+        self.columns.push(column);
+    }
+
+    pub fn apply_transform(&mut self, transform: &Transform) -> Result<()> {
+        match transform {
+            Transform::From(t) => {
+                *self = Frame::default();
+
+                let table_id = t
+                    .declared_at
+                    .ok_or_else(|| anyhow!("unresolved table {t:?}"))?;
+                self.tables.push(table_id);
+                self.columns.push(FrameColumn::All(table_id));
+            }
+
+            Transform::Select(select) => {
+                self.columns.clear();
+
+                self.apply_assigns(&select.assigns);
+            }
+            Transform::Derive(select) => {
+                self.apply_assigns(&select.assigns);
+            }
+            Transform::Group { pipeline, .. } => {
+                let pipeline = pipeline.item.as_pipeline().unwrap();
+                for transform in pipeline.as_transforms().unwrap() {
+                    self.apply_transform(transform)?;
+                }
+            }
+            Transform::Aggregate(select) => {
+                self.columns.clear();
+
+                for by in &select.group {
+                    self.push_column(None, by.declared_at.unwrap());
+                }
+
+                self.apply_assigns(&select.assigns);
+            }
+            Transform::Join { with, filter, .. } => {
+                let table_id = with
+                    .declared_at
+                    .ok_or_else(|| anyhow!("unresolved table {with:?}"))?;
+                self.tables.push(table_id);
+                self.columns.push(FrameColumn::All(table_id));
+
+                match filter {
+                    crate::ast::JoinFilter::On(_) => {}
+                    crate::ast::JoinFilter::Using(nodes) => {
+                        for node in nodes {
+                            let name = node.item.as_ident().unwrap().clone();
+                            let id = node.declared_at.unwrap();
+                            self.push_column(Some(name), id);
+                        }
+                    }
+                }
+            }
+            Transform::Sort(sort) => {
+                self.sort = extract_sorts(sort)?;
+            }
+            Transform::Filter(_) | Transform::Take(_) => {}
+        }
+        Ok(())
+    }
+
+    pub fn apply_assigns(&mut self, assigns: &[Node]) {
+        for node in assigns {
+            match &node.item {
+                Item::Ident(name) => {
+                    let id = node.declared_at.unwrap();
+
+                    if name == "<unnamed>" {
+                        self.push_column(None, id);
+                    } else {
+                        self.push_column(Some(name.clone()), id);
+                    }
+                }
+                _ => unreachable!("assign must contain only idents after being resolved"),
+            }
+        }
+    }
+
+    pub fn get_column_names(&self, context: &Context) -> Vec<Option<String>> {
+        self.columns
+            .iter()
+            .map(|col| match col {
+                FrameColumn::All(namespace) => {
+                    let (table, _) = &context.declarations[*namespace];
+                    let table = table.as_table().map(|x| x.as_str()).unwrap_or("");
+                    Some(format!("{table}.*"))
+                }
+                FrameColumn::Unnamed(_) => None,
+                FrameColumn::Named(name, _) => Some(name.clone()),
+            })
+            .collect()
+    }
+}
+
+fn extract_sorts(sort: &[ColumnSort]) -> Result<Vec<ColumnSort<usize>>> {
+    sort.iter()
+        .map(|s| {
+            Ok(ColumnSort {
+                column: (s.column.declared_at)
+                    .ok_or_else(|| anyhow!("Unresolved ident in sort?"))?,
+                direction: s.direction.clone(),
+            })
+        })
+        .try_collect()
+}
+
+impl Debug for Frame {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "[")?;
+        for t_col in &self.columns {
+            match t_col {
+                FrameColumn::All(ns) => write!(f, " {ns}.* ")?,
+                FrameColumn::Named(name, id) => write!(f, " {name}:{id} ")?,
+                FrameColumn::Unnamed(id) => write!(f, " {id} ")?,
+            }
+        }
+        writeln!(f, "]")
+    }
+}
+
+impl PartialEq<usize> for FrameColumn {
+    fn eq(&self, other: &usize) -> bool {
+        match self {
+            FrameColumn::All(_) => false,
+            FrameColumn::Unnamed(id) | FrameColumn::Named(_, id) => id == other,
+        }
+    }
+}
diff --git a/prql-compiler/src/semantic/mod.rs b/prql-compiler/src/semantic/mod.rs
index 620a3b75..cfe6f013 100644
--- a/prql-compiler/src/semantic/mod.rs
+++ b/prql-compiler/src/semantic/mod.rs
@@ -1,8 +1,12 @@
-mod cast_transforms;
 mod context;
+mod frame;
 mod reporting;
 mod resolver;
+mod scope;
+mod transforms;
 
-pub use self::context::{split_var_name, Context, Declaration, Frame, TableColumn};
+pub use self::context::{Context, Declaration};
+pub use self::frame::{Frame, FrameColumn};
+pub use self::scope::{split_var_name, Scope};
 pub use reporting::{collect_frames, label_references};
 pub use resolver::resolve;
diff --git a/prql-compiler/src/semantic/reporting.rs b/prql-compiler/src/semantic/reporting.rs
index cbca2a12..356de9fb 100644
--- a/prql-compiler/src/semantic/reporting.rs
+++ b/prql-compiler/src/semantic/reporting.rs
@@ -1,6 +1,6 @@
 use std::ops::Range;
 
-use anyhow::{Ok, Result};
+use anyhow::{anyhow, Ok, Result};
 use ariadne::{Color, Label, Report, ReportBuilder, ReportKind, Source};
 
 use super::{Context, Declaration, Frame};
@@ -60,9 +60,13 @@ impl<'a> AstFold for Labeler<'a> {
             let message = if let Some(span) = span {
                 let span = self.source.get_line_range(&Range::from(*span));
                 if span.len() <= 1 {
-                    format!("{declaration} at line {}", span.start + 1)
+                    format!("[{declared_at}] {declaration} at line {}", span.start + 1)
                 } else {
-                    format!("{declaration} at lines {}-{}", span.start + 1, span.end)
+                    format!(
+                        "[{declared_at}] {declaration} at lines {}-{}",
+                        span.start + 1,
+                        span.end
+                    )
                 }
             } else {
                 declaration.to_string()
@@ -100,12 +104,26 @@ struct FrameCollector {
 }
 
 impl AstFold for FrameCollector {
-    fn fold_node(&mut self, node: Node) -> Result<Node> {
-        if let Some(frame) = node.frame {
-            if let Some(span) = node.span {
-                self.frames.push((span, frame));
-            }
+    fn fold_table(&mut self, table: Table) -> Result<Table> {
+        Ok(table)
+    }
+
+    fn fold_pipeline(&mut self, pipeline: Pipeline) -> Result<Pipeline> {
+        let mut frame = Frame::default();
+        for function in &pipeline.functions {
+            let transform = (function.item)
+                .as_transform()
+                .ok_or_else(|| anyhow!("plain function in pipeline"))?;
+
+            frame.apply_transform(transform)?;
+
+            self.frames.push((function.span.unwrap(), frame.clone()));
         }
-        Ok(self.fold_item(node.item)?.into())
+
+        Ok(pipeline)
+    }
+
+    fn fold_func_def(&mut self, function: FuncDef) -> Result<FuncDef> {
+        fold_func_def(self, function)
     }
 }
diff --git a/prql-compiler/src/semantic/resolver.rs b/prql-compiler/src/semantic/resolver.rs
index 3990515d..a320d970 100644
--- a/prql-compiler/src/semantic/resolver.rs
+++ b/prql-compiler/src/semantic/resolver.rs
@@ -1,14 +1,14 @@
 use std::collections::HashSet;
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use itertools::Itertools;
 
 use crate::ast::ast_fold::*;
 use crate::error::{Error, Reason, WithErrorInfo};
-use crate::{ast::*, Declaration, TableColumn};
+use crate::{ast::*, Declaration};
 
-use super::cast_transforms;
-use super::context::{Context, Frame};
+use super::transforms;
+use super::Context;
 
 /// Runs semantic analysis on the query, using current state.
 /// Appends query to current query.
@@ -31,12 +31,15 @@ pub struct Resolver {
 
     /// True iff resolving a function curry (in a pipeline)
     within_curry: bool,
+
+    within_group: Vec<usize>,
 }
 impl Resolver {
     fn new(context: Context) -> Self {
         Resolver {
             context,
             within_curry: false,
+            within_group: vec![],
         }
     }
 }
@@ -73,7 +76,7 @@ impl AstFold for Resolver {
                         func_def.body = Box::new(self.fold_node(*func_def.body)?);
 
                         // clear declared variables
-                        self.context.clear_scope();
+                        self.context.scope.clear();
 
                         self.context.declare_func(func_def);
                         None
@@ -92,7 +95,7 @@ impl AstFold for Resolver {
         node.item = match node.item {
             Item::FuncCall(func_call) => {
                 // find declaration
-                node.declared_at = self.context.functions.get(&func_call.name).cloned();
+                node.declared_at = self.context.scope.functions.get(&func_call.name).cloned();
 
                 // validate function call
                 let (func_call, func_def) = self
@@ -101,13 +104,9 @@ impl AstFold for Resolver {
 
                 // fold (and cast if this is a transform)
                 if let Some(FuncKind::Transform) = func_def.kind {
-                    let transform = cast_transforms::cast_transform(func_call, node.span)?;
-
-                    let transform = self.fold_transform(transform)?;
-
-                    node.frame = Some(self.context.frame.clone());
+                    let transform = transforms::cast_transform(func_call, node.span)?;
 
-                    Item::Transform(transform)
+                    Item::Transform(self.fold_transform(transform)?)
                 } else {
                     Item::FuncCall(self.fold_func_call(func_call)?)
                 }
@@ -130,72 +129,65 @@ impl AstFold for Resolver {
     }
 
     fn fold_transform(&mut self, t: Transform) -> Result<Transform> {
-        Ok(match t {
-            Transform::From(t) => {
-                self.context.clear_scope();
+        let t = match t {
+            Transform::From(mut t) => {
+                self.context.scope.clear();
 
-                self.context.frame = Frame::default();
+                self.context.declare_table(&mut t);
 
-                self.context.declare_table(&t);
-
-                let t = Transform::From(t);
-                fold_transform(self, t)?
+                Transform::From(t)
             }
 
             Transform::Select(mut select) => {
-                self.context.frame.columns.clear();
-
                 select.assigns = self.fold_assigns(select.assigns)?;
-                self.apply_context(&mut select)?;
-
-                self.context.clear_scope();
+                self.context.scope.clear();
 
+                self.apply_context(&mut select)?;
                 Transform::Select(select)
             }
             Transform::Derive(mut select) => {
                 select.assigns = self.fold_assigns(select.assigns)?;
-                self.apply_context(&mut select)?;
 
+                self.apply_context(&mut select)?;
                 Transform::Derive(select)
             }
             Transform::Group { by, pipeline } => {
                 let by = self.fold_nodes(by)?;
-                self.context.frame.group = extract_group_by(&by);
 
+                self.within_group = by.iter().filter_map(|n| n.declared_at).collect();
                 let pipeline = Box::new(self.fold_node(*pipeline)?);
+                self.within_group = vec![];
 
-                self.context.frame.group.clear();
                 Transform::Group { by, pipeline }
             }
             Transform::Aggregate(mut select) => {
-                self.context.frame.columns.clear();
-                self.context.frame.groups_to_columns();
-
                 select.assigns = self.fold_assigns(select.assigns)?;
-                self.apply_context(&mut select)?;
-
-                self.context.clear_scope();
+                self.context.scope.clear();
 
+                self.apply_context(&mut select)?;
                 Transform::Aggregate(select)
             }
-            Transform::Join { side, with, filter } => {
-                self.context.declare_table(&with);
+            Transform::Join {
+                side,
+                mut with,
+                filter,
+            } => {
+                self.context.declare_table(&mut with);
 
                 Transform::Join {
                     side,
-                    with: self.fold_table_ref(with)?,
+                    with,
                     filter: self.fold_join_filter(filter)?,
                 }
             }
-            Transform::Sort(sort) => {
-                let sort = self.fold_column_sorts(sort)?;
+            t => fold_transform(self, t)?,
+        };
 
-                self.context.frame.sort = extract_sorts(sort.clone())?;
+        // if !self.within_group {
+        //     self.context.frame.apply_transform(&t)?;
+        // }
 
-                Transform::Sort(sort)
-            }
-            t => fold_transform(self, t)?,
-        })
+        Ok(t)
     }
 
     fn fold_join_filter(&mut self, filter: JoinFilter) -> Result<JoinFilter> {
@@ -220,10 +212,7 @@ impl AstFold for Resolver {
                     };
 
                     let id = self.context.declare(decl, node.span);
-                    self.context.add_to_scope(ident, id);
-
-                    let column = TableColumn::Named(ident.clone(), id);
-                    self.context.frame.columns.push(column);
+                    self.context.scope.add(ident.clone(), id);
 
                     node.declared_at = Some(id);
                 }
@@ -231,6 +220,17 @@ impl AstFold for Resolver {
             }
         })
     }
+
+    fn fold_table(&mut self, mut table: Table) -> Result<Table> {
+        // fold pipeline
+        table.pipeline = Box::new(self.fold_node(*table.pipeline)?);
+
+        // declare table
+        let decl = Declaration::Table(table.name.clone());
+        table.id = Some(self.context.declare(decl, None));
+
+        Ok(table)
+    }
 }
 
 impl Resolver {
@@ -238,57 +238,38 @@ impl Resolver {
         nodes
             .into_iter()
             .map(|mut node| {
-                match node.item {
+                Ok(match node.item {
                     Item::NamedExpr(NamedExpr { name, expr }) => {
                         // introduce a new expression alias
 
-                        let (expr, _) = self.fold_assign_expr(*expr)?;
+                        let expr = self.fold_assign_expr(*expr)?;
                         let id = expr.declared_at.unwrap();
 
-                        let value = TableColumn::Named(name.clone(), id);
-                        self.context.frame.columns.push(value);
-
-                        self.context.add_to_scope(&name, id);
+                        self.context.scope.add(name.clone(), id);
 
                         node.item = Item::Ident(name);
                         node.declared_at = Some(id);
-                        Ok(node)
+                        node
                     }
                     _ => {
-                        // try to guess a name, otherwise use unnamed column
-
-                        let (expr, name) = self.fold_assign_expr(node)?;
-                        let id = expr.declared_at.unwrap();
-
-                        let column = if let Some(name) = name {
-                            TableColumn::Named(name, id)
-                        } else {
-                            TableColumn::Unnamed(id)
-                        };
-
-                        self.context.frame.columns.push(column);
-                        Ok(expr)
+                        // no new names, only fold the expr
+                        self.fold_assign_expr(node)?
                     }
-                }
+                })
             })
             .try_collect()
     }
 
-    fn fold_assign_expr(&mut self, node: Node) -> Result<(Node, Option<String>)> {
+    fn fold_assign_expr(&mut self, node: Node) -> Result<Node> {
         let span = node.span;
 
-        match node.item {
-            Item::Ident(ref ident) => {
+        Ok(match node.item {
+            Item::Ident(_) => {
                 // keep existing ident
-
-                let name = ident.clone();
-                let node = self.fold_node(node)?;
-
-                Ok((node, Some(name)))
+                self.fold_node(node)?
             }
             _ => {
-                // declare new expression
-
+                // declare new expression so it can be references from FrameColumn
                 let expr = self.fold_node(node)?;
                 let decl = Declaration::Expression(Box::from(expr));
 
@@ -296,13 +277,13 @@ impl Resolver {
 
                 let mut placeholder: Node = Item::Ident("<unnamed>".to_string()).into();
                 placeholder.declared_at = Some(id);
-                Ok((placeholder, None))
+                placeholder
             }
-        }
+        })
     }
 
     fn apply_context(&self, select: &mut Select) -> Result<()> {
-        select.group = (self.context.frame.group)
+        select.group = (self.within_group)
             .iter()
             .map(|id| {
                 let mut node: Node = Item::Ident("<un-materialized>".to_string()).into();
@@ -310,7 +291,6 @@ impl Resolver {
                 node
             })
             .collect();
-
         Ok(())
     }
 
@@ -375,22 +355,6 @@ impl Resolver {
     }
 }
 
-fn extract_group_by(nodes: &[Node]) -> Vec<usize> {
-    nodes.iter().map(|n| n.declared_at.unwrap()).collect()
-}
-
-fn extract_sorts(sort: Vec<ColumnSort>) -> Result<Vec<ColumnSort<usize>>> {
-    sort.into_iter()
-        .map(|s| {
-            Ok(ColumnSort {
-                column: (s.column.declared_at)
-                    .ok_or_else(|| anyhow!("Unresolved ident in sort?"))?,
-                direction: s.direction,
-            })
-        })
-        .try_collect()
-}
-
 /// Loads `internal.prql` which contains type definitions of transforms
 pub fn init_context() -> Context {
     use crate::parse;
@@ -403,7 +367,7 @@ pub fn init_context() -> Context {
 
 #[cfg(test)]
 mod tests {
-    use insta::{assert_snapshot, assert_yaml_snapshot};
+    use insta::assert_snapshot;
     use serde_yaml::from_str;
 
     use crate::{parse, resolve_and_translate};
@@ -431,16 +395,7 @@ mod tests {
         .unwrap();
         resolver.fold_node(pipeline).unwrap();
 
-        assert_yaml_snapshot!(resolver.context.frame, @r###"
-        ---
-        columns:
-          - All: 30
-        sort: []
-        group: []
-        tables:
-          - 30
-        "###);
-        assert!(resolver.context.variables["employees.*"].len() == 1);
+        assert!(resolver.context.scope.variables["employees.*"].len() == 1);
     }
 
     #[test]
@@ -481,11 +436,9 @@ mod tests {
         .unwrap();
         resolver.fold_node(pipeline).unwrap();
 
-        assert_eq!(resolver.context.frame.columns.len(), 3);
-
-        assert!(resolver.context.variables.contains_key("salary_1"));
-        assert!(resolver.context.variables.contains_key("salary_2"));
-        assert!(resolver.context.variables.contains_key("age"));
+        assert!(resolver.context.scope.variables.contains_key("salary_1"));
+        assert!(resolver.context.scope.variables.contains_key("salary_2"));
+        assert!(resolver.context.scope.variables.contains_key("age"));
     }
 
     #[test]
@@ -506,7 +459,7 @@ mod tests {
         assert_snapshot!(result, @r###"
         SELECT
           salary AS salary1,
-          salary + 1 AS salary2,
+          salary1 + 1 AS salary2,
           age
         FROM
           employees
diff --git a/prql-compiler/src/semantic/scope.rs b/prql-compiler/src/semantic/scope.rs
new file mode 100644
index 00000000..92546095
--- /dev/null
+++ b/prql-compiler/src/semantic/scope.rs
@@ -0,0 +1,67 @@
+use std::collections::{HashMap, HashSet};
+
+use serde::{Deserialize, Serialize};
+
+pub const NS_PARAM: &str = "_param";
+const NS_GLOB: &str = "_glob";
+
+/// Maps from accessible names in some context to their declarations.
+#[derive(Default, Serialize, Deserialize, Clone)]
+pub struct Scope {
+    /// Mapping from idents to their declarations. For each namespace (table), a map from column names to their definitions
+    /// "_param" is namespace of current function parameters
+    /// "_glob" is namespace of functions without parameters (global variables)
+    pub(super) variables: HashMap<String, HashSet<usize>>,
+
+    pub(super) functions: HashMap<String, usize>,
+}
+
+impl Scope {
+    pub(super) fn add(&mut self, ident: String, id: usize) {
+        // insert into own namespace, override other declarations
+        let decls = self.variables.entry(ident.clone()).or_default();
+        decls.clear();
+        decls.insert(id);
+
+        self.cascade_variable(ident.as_str());
+    }
+
+    pub(super) fn add_function(&mut self, name: String, id: usize, has_params: bool) {
+        if has_params {
+            self.functions.insert(name, id);
+        } else {
+            self.add(format!("{NS_GLOB}.{name}"), id);
+        }
+    }
+
+    // insert into lower namespaces, possibly creating ambiguities
+    pub(super) fn cascade_variable(&mut self, ident: &str) {
+        let id = *self.variables[ident].iter().next().unwrap();
+
+        let (_, var_name) = split_var_name(ident);
+
+        let decls = self.variables.entry(var_name.to_string()).or_default();
+        decls.insert(id);
+    }
+
+    /// Removes all names from scope, except functions and columns in frame.
+    pub(super) fn clear(&mut self) {
+        let mut to_remove = HashSet::<usize>::new();
+        self.variables.retain(|name, decls| {
+            let remove = name.starts_with(NS_PARAM) || name.ends_with(".*") || name == "*";
+            if remove {
+                to_remove.extend(decls.iter());
+            }
+            !remove
+        });
+        self.variables.retain(|_, decls| {
+            decls.retain(|d| !to_remove.contains(d));
+            !decls.is_empty()
+        });
+    }
+}
+
+/// Splits ident into namespaces and variable name
+pub fn split_var_name(ident: &str) -> (&str, &str) {
+    ident.rsplit_once('.').unwrap_or(("", ident))
+}
diff --git a/prql-compiler/src/semantic/cast_transforms.rs b/prql-compiler/src/semantic/transforms.rs
similarity index 94%
rename from prql-compiler/src/semantic/cast_transforms.rs
rename to prql-compiler/src/semantic/transforms.rs
index 2c35cd33..138513c5 100644
--- a/prql-compiler/src/semantic/cast_transforms.rs
+++ b/prql-compiler/src/semantic/transforms.rs
@@ -16,6 +16,7 @@ pub fn cast_transform(func_call: FuncCall, span: Option<Span>) -> Result<Transfo
                     "`from` does not support inline expressions. You can only pass a table name.",
                 )?,
                 alias: name,
+                declared_at: None,
             };
 
             Transform::From(table_ref)
@@ -23,28 +24,28 @@ pub fn cast_transform(func_call: FuncCall, span: Option<Span>) -> Result<Transfo
         "select" => {
             let ([assigns], []) = unpack(func_call, [])?;
 
-            Transform::Select(Select::new(assigns.coerce_to_items()))
+            Transform::Select(Select::new(assigns.coerce_to_vec()))
         }
         "filter" => {
             let ([filter], []) = unpack(func_call, [])?;
 
-            Transform::Filter(filter.coerce_to_items())
+            Transform::Filter(filter.coerce_to_vec())
         }
         "derive" => {
             let ([assigns], []) = unpack(func_call, [])?;
 
-            Transform::Derive(Select::new(assigns.coerce_to_items()))
+            Transform::Derive(Select::new(assigns.coerce_to_vec()))
         }
         "aggregate" => {
             let ([assigns], []) = unpack(func_call, [])?;
 
-            Transform::Aggregate(Select::new(assigns.coerce_to_items()))
+            Transform::Aggregate(Select::new(assigns.coerce_to_vec()))
         }
         "sort" => {
             let ([by], []) = unpack(func_call, [])?;
 
             let by = by
-                .coerce_to_items()
+                .coerce_to_vec()
                 .into_iter()
                 .map(|node| {
                     let (column, direction) = match node.item {
@@ -114,9 +115,10 @@ pub fn cast_transform(func_call: FuncCall, span: Option<Span>) -> Result<Transfo
                     "`join` does not support inline expressions. You can only pass a table name.",
                 )?,
                 alias: with_alias,
+                declared_at: None,
             };
 
-            let filter = filter.discard_name()?.coerce_to_items();
+            let filter = filter.discard_name()?.coerce_to_vec();
             let use_using = (filter.iter().map(|x| &x.item)).all(|x| matches!(x, Item::Ident(_)));
 
             let filter = if use_using {
@@ -131,7 +133,7 @@ pub fn cast_transform(func_call: FuncCall, span: Option<Span>) -> Result<Transfo
             let ([by, pipeline], []) = unpack(func_call, [])?;
 
             let by = by
-                .coerce_to_items()
+                .coerce_to_vec()
                 .into_iter()
                 // check that they are only idents
                 .map(|n| match n.item {
@@ -221,6 +223,7 @@ mod tests {
                   From:
                     name: c_invoice
                     alias: ~
+                    declared_at: 58
               - Transform:
                   Group:
                     by:
@@ -275,6 +278,7 @@ mod tests {
                   From:
                     name: c_invoice
                     alias: ~
+                    declared_at: 58
               - Transform:
                   Group:
                     by:
diff --git a/prql-compiler/src/sql/materializer.rs b/prql-compiler/src/sql/materializer.rs
index 1afa77bb..df244b59 100644
--- a/prql-compiler/src/sql/materializer.rs
+++ b/prql-compiler/src/sql/materializer.rs
@@ -3,36 +3,40 @@
 //! contains no query-specific logic.
 use crate::ast::ast_fold::*;
 use crate::ast::*;
+use crate::semantic;
 use crate::Declaration;
+use crate::Frame;
 
 use anyhow::{anyhow, Result};
 use itertools::zip;
 use itertools::Itertools;
 
-use crate::semantic::{split_var_name, Context, Frame, TableColumn};
-
-pub struct MaterializedFrame {
-    pub columns: Vec<Node>,
-    pub sort: Vec<ColumnSort<Ident>>,
-}
+use crate::semantic::{split_var_name, FrameColumn};
 
 /// Replaces all resolved functions and variables with their declarations.
 pub fn materialize(
     pipeline: Pipeline,
-    frame: Frame,
-    context: Context,
-    as_table: Option<&str>,
-) -> Result<(Pipeline, MaterializedFrame, Context)> {
-    let mut counter = TableCounter::default();
-    let pipeline = counter.fold_pipeline(pipeline).unwrap();
+    mut context: MaterializationContext,
+    as_table: Option<usize>,
+) -> Result<(Pipeline, MaterializedFrame, MaterializationContext)> {
+    // find frame
+    for transform in &pipeline.functions {
+        let transform = (transform.item)
+            .as_transform()
+            .ok_or_else(|| anyhow!("plain function in pipeline"))?;
+        context.frame.apply_transform(transform)?;
+    }
 
+    // init
+    let remove_namespaces = context.frame.tables.len() == 1;
     let mut m = Materializer {
         context,
-        remove_namespaces: counter.tables == 1,
+        remove_namespaces,
     };
 
     // materialize the query
     let pipeline = m.fold_pipeline(pipeline)?;
+    let frame = m.context.frame.clone();
 
     // materialize each of the columns
     let columns = m.anchor_columns(frame.columns, as_table)?;
@@ -42,15 +46,57 @@ pub fn materialize(
 
     // rename tables for future pipelines
     if let Some(as_table) = as_table {
-        m.rename_tables(frame.tables, as_table);
+        m.replace_tables(frame.tables, as_table);
     }
 
     Ok((pipeline, MaterializedFrame { columns, sort }, m.context))
 }
+pub struct MaterializedFrame {
+    pub columns: Vec<Node>,
+    pub sort: Vec<ColumnSort<Ident>>,
+}
+
+pub struct MaterializationContext {
+    /// Map of all accessible names (for each namespace)
+    pub(super) frame: Frame,
+
+    /// All declarations, mostly from [semantic::Context]
+    pub(super) declarations: Vec<Declaration>,
+}
+
+impl MaterializationContext {
+    pub fn declare_table(&mut self, table: &str) -> usize {
+        let decl = Declaration::Table(table.to_string());
+        self.declarations.push(decl);
+
+        let id = self.declarations.len() - 1;
+
+        self.frame.tables.push(id);
+        id
+    }
+
+    pub(crate) fn replace_declaration(&mut self, id: usize, new_decl: Declaration) {
+        let decl = self.declarations.get_mut(id).unwrap();
+        *decl = new_decl;
+    }
+
+    pub(crate) fn replace_declaration_expr(&mut self, id: usize, expr: Node) {
+        self.replace_declaration(id, Declaration::Expression(Box::new(expr)));
+    }
+}
+
+impl From<semantic::Context> for MaterializationContext {
+    fn from(context: semantic::Context) -> Self {
+        MaterializationContext {
+            frame: Frame::default(),
+            declarations: context.declarations.into_iter().map(|x| x.0).collect(),
+        }
+    }
+}
 
 /// Can fold (walk) over AST and replace function calls and variable references with their declarations.
 pub struct Materializer {
-    pub context: Context,
+    pub context: MaterializationContext,
     pub remove_namespaces: bool,
 }
 
@@ -58,20 +104,15 @@ impl Materializer {
     /// Looks up column declarations and replaces them with an identifiers.
     fn anchor_columns(
         &mut self,
-        columns: Vec<TableColumn>,
-        as_table: Option<&str>,
+        columns: Vec<FrameColumn>,
+        as_table: Option<usize>,
     ) -> Result<Vec<Node>> {
-        let as_table = as_table.map(|table| {
-            let decl = Declaration::Table(table.to_string());
-            self.context.declare(decl, None)
-        });
-
         columns
             .into_iter()
             .map(|column| {
                 Ok(match column {
-                    TableColumn::Named(name, id) => {
-                        let expr_node = self.lookup_declaration(id)?;
+                    FrameColumn::Named(name, id) => {
+                        let expr_node = self.materialize_declaration(id)?;
 
                         let name = split_var_name(&name).1.to_string();
 
@@ -81,27 +122,14 @@ impl Materializer {
                         };
                         self.context.replace_declaration(id, decl);
 
-                        // is expr_node just an ident with same name?
-                        let expr_ident = expr_node.item.as_ident().map(|n| split_var_name(n).1);
-
-                        if expr_ident.map(|n| n == name).unwrap_or(false) {
-                            // return just the ident
-                            expr_node
-                        } else {
-                            // return expr with new name
-                            Item::NamedExpr(NamedExpr {
-                                expr: Box::new(expr_node),
-                                name,
-                            })
-                            .into()
-                        }
+                        emit_column_with_name(expr_node, name)
                     }
-                    TableColumn::Unnamed(id) => {
+                    FrameColumn::Unnamed(id) => {
                         // no need to replace declaration, since it cannot be referenced again
-                        self.lookup_declaration(id)?
+                        self.materialize_declaration(id)?
                     }
-                    TableColumn::All(namespace) => {
-                        let (decl, _) = &self.context.declarations[namespace];
+                    FrameColumn::All(namespace) => {
+                        let decl = &self.context.declarations[namespace];
                         let table = decl.as_table().unwrap();
                         Item::Ident(format!("{table}.*")).into()
                     }
@@ -110,31 +138,19 @@ impl Materializer {
             .try_collect()
     }
 
-    fn rename_tables(&mut self, tables: Vec<usize>, new_table: &str) {
+    fn replace_tables(&mut self, tables: Vec<usize>, new_table: usize) {
+        let new_table = self.context.declarations[new_table].clone();
         for id in tables {
-            let (decl, _) = self.context.declarations.get_mut(id).unwrap();
-
-            let table = decl.as_table_mut().unwrap();
-            *table = new_table.to_string();
+            self.context.replace_declaration(id, new_table.clone());
         }
     }
 
-    fn lookup_declaration(&mut self, id: usize) -> Result<Node> {
-        let (decl, _) = &self.context.declarations[id];
-        if !matches!(decl, &Declaration::Expression { .. }) {
-            self.materialize_declaration(id)?;
-        }
-
-        let (decl, _) = &self.context.declarations[id];
-        Ok(*decl.clone().into_expression()?)
-    }
-
     /// Folds the column and returns expression that can be used in select.
     /// Also returns column id and name if declaration should be replaced.
     fn lookup_sort(&mut self, sort: Vec<ColumnSort<usize>>) -> Result<Vec<ColumnSort<Ident>>> {
         sort.into_iter()
             .map(|s| {
-                let ident = self.lookup_declaration(s.column)?;
+                let ident = self.materialize_declaration(s.column)?;
                 let column = ident.item.into_ident()?;
 
                 Ok(ColumnSort {
@@ -146,10 +162,7 @@ impl Materializer {
     }
 
     fn materialize_declaration(&mut self, id: usize) -> Result<Node> {
-        let (decl, _) = &self.context.declarations[id];
-
-        // eprintln!("materialize_declaration {id}");
-        // dbg!(decl);
+        let decl = &self.context.declarations[id];
 
         let materialized = match decl.clone() {
             Declaration::Expression(inner) => self.fold_node(*inner)?,
@@ -160,7 +173,7 @@ impl Materializer {
                     if self.remove_namespaces {
                         var_name.to_string()
                     } else {
-                        let (table, _) = &self.context.declarations[table];
+                        let table = &self.context.declarations[table];
                         let table = table.as_table().unwrap();
                         format!("{table}.{var_name}")
                     }
@@ -179,8 +192,6 @@ impl Materializer {
             }
             Declaration::Table(table) => Item::Ident(format!("{table}.*")).into(),
         };
-        self.context
-            .replace_declaration_expr(id, materialized.clone());
 
         Ok(materialized)
     }
@@ -190,7 +201,7 @@ impl Materializer {
 
         // locate declaration
         let func_dec = node.declared_at.ok_or_else(|| anyhow!("unresolved"))?;
-        let func_dec = &self.context.declarations[func_dec].0;
+        let func_dec = &self.context.declarations[func_dec];
         let func_dec = func_dec.as_function().unwrap().clone();
 
         // TODO: check if the function is called recursively.
@@ -218,6 +229,23 @@ impl Materializer {
     }
 }
 
+fn emit_column_with_name(expr_node: Node, name: String) -> Node {
+    // is expr_node just an ident with same name?
+    let expr_ident = expr_node.item.as_ident().map(|n| split_var_name(n).1);
+
+    if expr_ident.map(|n| n == name).unwrap_or(false) {
+        // return just the ident
+        expr_node
+    } else {
+        // return expr with new name
+        Item::NamedExpr(NamedExpr {
+            expr: Box::new(expr_node),
+            name,
+        })
+        .into()
+    }
+}
+
 impl AstFold for Materializer {
     fn fold_node(&mut self, mut node: Node) -> Result<Node> {
         // We replace Items and also pass node to `inline_func_call`,
@@ -275,29 +303,27 @@ impl AstFold for Materializer {
     }
 }
 
-/// Counts all tables in scope
-#[derive(Default)]
-struct TableCounter {
-    tables: usize,
-}
-
-impl AstFold for TableCounter {
-    fn fold_func_def(&mut self, function: FuncDef) -> Result<FuncDef> {
-        Ok(function)
-    }
-
-    fn fold_transform(&mut self, transform: Transform) -> Result<Transform> {
-        match &transform {
-            Transform::From(_) | Transform::Join { .. } => {
-                self.tables += 1;
+impl std::fmt::Debug for MaterializationContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for (i, d) in self.declarations.iter().enumerate() {
+            match d {
+                Declaration::Expression(v) => {
+                    writeln!(f, "[{i:3}]: expr  `{}`", v.item)?;
+                }
+                Declaration::ExternRef { table, variable } => {
+                    writeln!(f, "[{i:3}]: col   `{variable}` from table {table:?}")?;
+                }
+                Declaration::Table(name) => {
+                    writeln!(f, "[{i:3}]: table `{name}`")?;
+                }
+                Declaration::Function(func) => {
+                    writeln!(f, "[{i:3}]: func  `{}`", func.name)?;
+                }
             }
-            _ => {}
         }
-        // no need to recurse, transformations cannot be nested
-        Ok(transform)
+        Ok(())
     }
 }
-
 #[cfg(test)]
 mod test {
 
@@ -306,86 +332,63 @@ mod test {
     use insta::{assert_display_snapshot, assert_snapshot, assert_yaml_snapshot};
     use serde_yaml::to_string;
 
-    fn resolve_and_materialize(nodes: Vec<Node>) -> Result<Vec<Node>> {
-        let (res, context) = resolve(nodes, None)?;
+    fn find_pipeline(mut res: Vec<Node>) -> Pipeline {
+        res.remove(res.len() - 1).item.into_pipeline().unwrap()
+    }
+
+    fn resolve_and_materialize(query: Query) -> Result<Vec<Node>> {
+        let (res, context) = resolve(query.nodes, None)?;
 
-        let pipeline = res.last().unwrap().item.as_pipeline().unwrap();
-        let frame = pipeline.functions.last().unwrap().frame.clone().unwrap();
+        let pipeline = find_pipeline(res);
 
-        let (mat, _, _) = materialize(res.into(), frame, context, Some("table_1"))?;
+        let (mat, _, _) = materialize(pipeline, context.into(), None)?;
         Ok(mat.functions)
     }
 
     #[test]
     fn test_replace_variables_1() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"from employees
     derive [                                         # This adds columns / variables.
       gross_salary: salary + payroll_tax,
       gross_cost:   gross_salary + benefits_cost     # Variables can use other variables.
     ]
     "#,
-        )?
-        .nodes;
+        )?;
 
-        let mat = resolve_and_materialize(ast.clone()).unwrap();
+        let (res, context) = resolve(query.nodes, None)?;
+
+        let pipeline = find_pipeline(res);
+
+        let (mat, _, _) = materialize(pipeline.clone(), context.into(), None)?;
 
         // We could make a convenience function for this. It's useful for
         // showing the diffs of an operation.
         assert_display_snapshot!(diff(
-            &to_string(&ast)?,
+            &to_string(&pipeline)?,
             &to_string(&mat)?
         ),
         @r###"
-        @@ -2,27 +2,24 @@
-         - Pipeline:
-             value: ~
-             functions:
-        -      - FuncCall:
-        -          name: from
-        -          args:
-        -            - Ident: employees
-        -          named_args: {}
-        -      - FuncCall:
-        -          name: derive
-        -          args:
-        -            - List:
-        -                - NamedExpr:
-        -                    name: gross_salary
-        -                    expr:
-        -                      Expr:
-        -                        - Ident: salary
-        -                        - Raw: +
-        -                        - Ident: payroll_tax
-        -                - NamedExpr:
-        -                    name: gross_cost
-        -                    expr:
-        -                      Expr:
-        -                        - Ident: gross_salary
-        -                        - Raw: +
-        -                        - Ident: benefits_cost
-        -          named_args: {}
-        +      - Transform:
-        +          From:
-        +            name: employees
-        +            alias: ~
-        +      - Transform:
-        +          Derive:
-        +            assigns:
+        @@ -9,8 +9,17 @@
+           - Transform:
+               Derive:
+                 assigns:
+        -          - Ident: gross_salary
+        -          - Ident: gross_cost
+        +          - Expr:
+        +              - Ident: salary
+        +              - Raw: +
+        +              - Ident: payroll_tax
+        +          - Expr:
         +              - Expr:
         +                  - Ident: salary
         +                  - Raw: +
         +                  - Ident: payroll_tax
-        +              - Expr:
-        +                  - Expr:
-        +                      - Ident: salary
-        +                      - Raw: +
-        +                      - Ident: payroll_tax
-        +                  - Raw: +
-        +                  - Ident: benefits_cost
-        +            group: []
-        +            window: ~
-        +            sort: ~
+        +              - Raw: +
+        +              - Ident: benefits_cost
+                 group: []
+                 window: ~
+                 sort: ~
         "###);
 
         Ok(())
@@ -393,7 +396,7 @@ mod test {
 
     #[test]
     fn test_replace_variables_2() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
 func count = s"COUNT(*)"
 func average column = s"AVG({column})"
@@ -421,9 +424,9 @@ sort sum_gross_cost
 filter sum_gross_cost > 200
 take 20
 "#,
-        )?.nodes;
+        )?;
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(&mat);
 
         Ok(())
@@ -431,7 +434,7 @@ take 20
 
     #[test]
     fn test_run_functions_args() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
         func count x = s"count({x})"
 
@@ -440,10 +443,9 @@ take 20
         count salary
         ]
         "#,
-        )?
-        .nodes;
+        )?;
 
-        assert_yaml_snapshot!(ast, @r###"
+        assert_yaml_snapshot!(query.nodes, @r###"
         ---
         - FuncDef:
             name: count
@@ -477,60 +479,33 @@ take 20
                   named_args: {}
         "###);
 
-        let mat = resolve_and_materialize(ast.clone()).unwrap();
+        let (res, context) = resolve(query.nodes, None)?;
+
+        let pipeline = find_pipeline(res);
+
+        let (mat, _, _) = materialize(pipeline.clone(), context.into(), None)?;
 
         // We could make a convenience function for this. It's useful for
         // showing the diffs of an operation.
-        let diff = diff(&to_string(&ast)?, &to_string(&mat)?);
+        let diff = diff(
+            &to_string(&pipeline.functions)?,
+            &to_string(&mat.functions)?,
+        );
         assert!(!diff.is_empty());
         assert_display_snapshot!(diff, @r###"
-        @@ -1,31 +1,19 @@
-         ---
-        -- FuncDef:
-        -    name: count
-        -    kind: ~
-        -    positional_params:
-        -      - Ident: x
-        -    named_params: []
-        -    body:
-        -      SString:
-        -        - String: count(
-        -        - Expr:
-        -            Ident: x
-        -        - String: )
-         - Pipeline:
-             value: ~
-             functions:
-        -      - FuncCall:
-        -          name: from
-        -          args:
-        -            - Ident: employees
-        -          named_args: {}
-        -      - FuncCall:
-        -          name: aggregate
-        -          args:
-        -            - List:
-        -                - FuncCall:
-        -                    name: count
-        -                    args:
-        -                      - Ident: salary
-        -                    named_args: {}
-        -          named_args: {}
-        +      - Transform:
-        +          From:
-        +            name: employees
-        +            alias: ~
-        +      - Transform:
-        +          Aggregate:
-        +            assigns:
-        +              - SString:
-        +                  - String: count(
-        +                  - Expr:
-        +                      Ident: salary
-        +                  - String: )
-        +            group: []
-        +            window: ~
-        +            sort: ~
+        @@ -7,7 +7,11 @@
+         - Transform:
+             Aggregate:
+               assigns:
+        -        - Ident: "<unnamed>"
+        +        - SString:
+        +            - String: count(
+        +            - Expr:
+        +                Ident: salary
+        +            - String: )
+               group: []
+               window: ~
+               sort: ~
         "###);
 
         Ok(())
@@ -538,7 +513,7 @@ take 20
 
     #[test]
     fn test_run_functions_nested() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
         func lag_day x = s"lag_day_todo({x})"
         func ret x dividend_return = x / (lag_day x) - 1 + dividend_return
@@ -546,10 +521,9 @@ take 20
         from a
         select (ret b c)
         "#,
-        )?
-        .nodes;
+        )?;
 
-        assert_yaml_snapshot!(ast[2], @r###"
+        assert_yaml_snapshot!(query.nodes[2], @r###"
         ---
         Pipeline:
           value: ~
@@ -571,35 +545,33 @@ take 20
                 named_args: {}
         "###);
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(mat, @r###"
         ---
-        - Pipeline:
-            value: ~
-            functions:
-              - Transform:
-                  From:
-                    name: a
-                    alias: ~
-              - Transform:
-                  Select:
-                    assigns:
-                      - Expr:
-                          - Expr:
-                              - Ident: b
-                              - Raw: /
-                              - SString:
-                                  - String: lag_day_todo(
-                                  - Expr:
-                                      Ident: b
-                                  - String: )
-                          - Raw: "-"
-                          - Raw: "1"
-                          - Raw: +
-                          - Ident: c
-                    group: []
-                    window: ~
-                    sort: ~
+        - Transform:
+            From:
+              name: a
+              alias: ~
+              declared_at: 35
+        - Transform:
+            Select:
+              assigns:
+                - Expr:
+                    - Expr:
+                        - Ident: b
+                        - Raw: /
+                        - SString:
+                            - String: lag_day_todo(
+                            - Expr:
+                                Ident: b
+                            - String: )
+                    - Raw: "-"
+                    - Raw: "1"
+                    - Raw: +
+                    - Ident: c
+              group: []
+              window: ~
+              sort: ~
         "###);
 
         Ok(())
@@ -607,92 +579,45 @@ take 20
 
     #[test]
     fn test_run_inline_pipelines() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
         func sum x = s"SUM({x})"
 
         from a
         aggregate [one: (foo | sum), two: (foo | sum)]
         "#,
-        )?
-        .nodes;
-
-        let mat = resolve_and_materialize(ast.clone()).unwrap();
-
-        assert_snapshot!(diff(&to_string(&ast)?, &to_string(&mat)?), @r###"
-        @@ -1,48 +1,24 @@
-         ---
-        -- FuncDef:
-        -    name: sum
-        -    kind: ~
-        -    positional_params:
-        -      - Ident: x
-        -    named_params: []
-        -    body:
-        -      SString:
-        -        - String: SUM(
-        -        - Expr:
-        -            Ident: x
-        -        - String: )
-         - Pipeline:
-             value: ~
-             functions:
-        -      - FuncCall:
-        -          name: from
-        -          args:
-        -            - Ident: a
-        -          named_args: {}
-        -      - FuncCall:
-        -          name: aggregate
-        -          args:
-        -            - List:
-        -                - NamedExpr:
-        -                    name: one
-        -                    expr:
-        -                      Pipeline:
-        -                        value:
-        -                          Ident: foo
-        -                        functions:
-        -                          - FuncCall:
-        -                              name: sum
-        -                              args: []
-        -                              named_args: {}
-        -                - NamedExpr:
-        -                    name: two
-        -                    expr:
-        -                      Pipeline:
-        -                        value:
-        -                          Ident: foo
-        -                        functions:
-        -                          - FuncCall:
-        -                              name: sum
-        -                              args: []
-        -                              named_args: {}
-        -          named_args: {}
-        +      - Transform:
-        +          From:
-        +            name: a
-        +            alias: ~
-        +      - Transform:
-        +          Aggregate:
-        +            assigns:
-        +              - SString:
-        +                  - String: SUM(
-        +                  - Expr:
-        +                      Ident: foo
-        +                  - String: )
-        +              - SString:
-        +                  - String: SUM(
-        +                  - Expr:
-        +                      Ident: foo
-        +                  - String: )
-        +            group: []
-        +            window: ~
-        +            sort: ~
+        )?;
+
+        let (res, context) = resolve(query.nodes, None)?;
+
+        let pipeline = find_pipeline(res);
+
+        let (mat, _, _) = materialize(pipeline.clone(), context.into(), None)?;
+
+        assert_snapshot!(diff(&to_string(&pipeline.functions)?, &to_string(&mat.functions)?), @r###"
+        @@ -7,8 +7,16 @@
+         - Transform:
+             Aggregate:
+               assigns:
+        -        - Ident: one
+        -        - Ident: two
+        +        - SString:
+        +            - String: SUM(
+        +            - Expr:
+        +                Ident: foo
+        +            - String: )
+        +        - SString:
+        +            - String: SUM(
+        +            - Expr:
+        +                Ident: foo
+        +            - String: )
+               group: []
+               window: ~
+               sort: ~
         "###);
 
         // Test it'll run the `sum foo` function first.
-        let ast = parse(
+        let query = parse(
             r#"
         func sum x = s"SUM({x})"
         func plus_one x = x + 1
@@ -700,34 +625,31 @@ take 20
         from a
         aggregate [a: (sum foo | plus_one)]
         "#,
-        )?
-        .nodes;
+        )?;
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
 
         assert_yaml_snapshot!(mat, @r###"
         ---
-        - Pipeline:
-            value: ~
-            functions:
-              - Transform:
-                  From:
-                    name: a
-                    alias: ~
-              - Transform:
-                  Aggregate:
-                    assigns:
-                      - Expr:
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Ident: foo
-                              - String: )
-                          - Raw: +
-                          - Raw: "1"
-                    group: []
-                    window: ~
-                    sort: ~
+        - Transform:
+            From:
+              name: a
+              alias: ~
+              declared_at: 34
+        - Transform:
+            Aggregate:
+              assigns:
+                - Expr:
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Ident: foo
+                        - String: )
+                    - Raw: +
+                    - Raw: "1"
+              group: []
+              window: ~
+              sort: ~
         "###);
 
         Ok(())
@@ -735,7 +657,7 @@ take 20
 
     #[test]
     fn test_named_args() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
         func add x to:1  = x + to
 
@@ -745,33 +667,30 @@ take 20
         added_default: add bar
         ]
         "#,
-        )?
-        .nodes;
-        let mat = resolve_and_materialize(ast).unwrap();
+        )?;
+        let mat = resolve_and_materialize(query).unwrap();
 
         assert_yaml_snapshot!(mat, @r###"
         ---
-        - Pipeline:
-            value: ~
-            functions:
-              - Transform:
-                  From:
-                    name: foo_table
-                    alias: ~
-              - Transform:
-                  Derive:
-                    assigns:
-                      - Expr:
-                          - Ident: bar
-                          - Raw: +
-                          - Raw: "3"
-                      - Expr:
-                          - Ident: bar
-                          - Raw: +
-                          - Raw: "1"
-                    group: []
-                    window: ~
-                    sort: ~
+        - Transform:
+            From:
+              name: foo_table
+              alias: ~
+              declared_at: 33
+        - Transform:
+            Derive:
+              assigns:
+                - Expr:
+                    - Ident: bar
+                    - Raw: +
+                    - Raw: "3"
+                - Expr:
+                    - Ident: bar
+                    - Raw: +
+                    - Raw: "1"
+              group: []
+              window: ~
+              sort: ~
         "###);
 
         Ok(())
@@ -779,7 +698,7 @@ take 20
 
     #[test]
     fn test_materialize_1() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
         func count x = s"count({x})"
 
@@ -788,31 +707,28 @@ take 20
         count salary
         ]
         "#,
-        )?
-        .nodes;
+        )?;
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(mat,
             @r###"
         ---
-        - Pipeline:
-            value: ~
-            functions:
-              - Transform:
-                  From:
-                    name: employees
-                    alias: ~
-              - Transform:
-                  Aggregate:
-                    assigns:
-                      - SString:
-                          - String: count(
-                          - Expr:
-                              Ident: salary
-                          - String: )
-                    group: []
-                    window: ~
-                    sort: ~
+        - Transform:
+            From:
+              name: employees
+              alias: ~
+              declared_at: 32
+        - Transform:
+            Aggregate:
+              assigns:
+                - SString:
+                    - String: count(
+                    - Expr:
+                        Ident: salary
+                    - String: )
+              group: []
+              window: ~
+              sort: ~
         "###
         );
         Ok(())
@@ -820,7 +736,7 @@ take 20
 
     #[test]
     fn test_materialize_2() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
 func count = s"COUNT(*)"
 func average column = s"AVG({column})"
@@ -848,43 +764,42 @@ sort sum_gross_cost
 filter sum_gross_cost > 200
 take 20
 "#,
-        )?.nodes;
+        )?;
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(mat);
         Ok(())
     }
 
     #[test]
     fn test_materialize_3() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
-    func interest_rate = 0.2
-
-    func lag_day x = s"lag_day_todo({x})"
-    func ret x dividend_return = x / (lag_day x) - 1 + dividend_return
-    func excess x = (x - interest_rate) / 252
-    func if_valid x = s"IF(is_valid_price, {x}, NULL)"
-
-    from prices
-    derive [
-      return_total     : if_valid (ret prices_adj div_ret),
-      return_usd       : if_valid (ret prices_usd div_ret),
-      return_excess    : excess return_total,
-      return_usd_excess: excess return_usd,
-    ]
-    select [
-      date,
-      sec_id,
-      return_total,
-      return_usd,
-      return_excess,
-      return_usd_excess,
-    ]
-    "#,
-        )?
-        .nodes;
-        let mat = resolve_and_materialize(ast).unwrap();
+        func interest_rate = 0.2
+
+        func lag_day x = s"lag_day_todo({x})"
+        func ret x dividend_return = x / (lag_day x) - 1 + dividend_return
+        func excess x = (x - interest_rate) / 252
+        func if_valid x = s"IF(is_valid_price, {x}, NULL)"
+
+        from prices
+        derive [
+        return_total     : if_valid (ret prices_adj div_ret),
+        return_usd       : if_valid (ret prices_usd div_ret),
+        return_excess    : excess return_total,
+        return_usd_excess: excess return_usd,
+        ]
+        select [
+        date,
+        sec_id,
+        return_total,
+        return_usd,
+        return_excess,
+        return_usd_excess,
+        ]
+        "#,
+        )?;
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(mat);
 
         Ok(())
@@ -892,77 +807,160 @@ take 20
 
     #[test]
     fn test_variable_after_aggregate() -> Result<()> {
-        let ast = parse(
+        let query = parse(
             r#"
-func average column = s"AVG({column})"
+        func average column = s"AVG({column})"
 
-from employees
-group [title, emp_no] (
-    aggregate [emp_salary: average salary]
-)
-group [title] (
-    aggregate [avg_salary: average emp_salary]
-)
-"#,
-        )?
-        .nodes;
+        from employees
+        group [title, emp_no] (
+            aggregate [emp_salary: average salary]
+        )
+        group [title] (
+            aggregate [avg_salary: average emp_salary]
+        )
+        "#,
+        )?;
 
-        let mat = resolve_and_materialize(ast).unwrap();
+        let mat = resolve_and_materialize(query).unwrap();
         assert_yaml_snapshot!(mat, @r###"
         ---
-        - Pipeline:
-            value: ~
-            functions:
-              - Transform:
-                  From:
-                    name: employees
-                    alias: ~
-              - Transform:
-                  Group:
-                    by:
-                      - Ident: title
-                      - Ident: emp_no
-                    pipeline:
-                      Pipeline:
-                        value: ~
-                        functions:
-                          - Transform:
-                              Aggregate:
-                                assigns:
-                                  - SString:
+        - Transform:
+            From:
+              name: employees
+              alias: ~
+              declared_at: 32
+        - Transform:
+            Group:
+              by:
+                - Ident: title
+                - Ident: emp_no
+              pipeline:
+                Pipeline:
+                  value: ~
+                  functions:
+                    - Transform:
+                        Aggregate:
+                          assigns:
+                            - SString:
+                                - String: AVG(
+                                - Expr:
+                                    Ident: salary
+                                - String: )
+                          group:
+                            - Ident: title
+                            - Ident: emp_no
+                          window: ~
+                          sort: ~
+        - Transform:
+            Group:
+              by:
+                - Ident: title
+              pipeline:
+                Pipeline:
+                  value: ~
+                  functions:
+                    - Transform:
+                        Aggregate:
+                          assigns:
+                            - SString:
+                                - String: AVG(
+                                - Expr:
+                                    SString:
                                       - String: AVG(
                                       - Expr:
                                           Ident: salary
                                       - String: )
-                                group:
-                                  - Ident: title
-                                  - Ident: emp_no
-                                window: ~
-                                sort: ~
-              - Transform:
-                  Group:
-                    by:
-                      - Ident: title
-                    pipeline:
-                      Pipeline:
-                        value: ~
-                        functions:
-                          - Transform:
-                              Aggregate:
-                                assigns:
-                                  - SString:
-                                      - String: AVG(
-                                      - Expr:
-                                          SString:
-                                            - String: AVG(
-                                            - Expr:
-                                                Ident: salary
-                                            - String: )
-                                      - String: )
-                                group:
-                                  - Ident: title
-                                window: ~
-                                sort: ~
+                                - String: )
+                          group:
+                            - Ident: title
+                          window: ~
+                          sort: ~
+        "###);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_frames_and_names() -> Result<()> {
+        let query1 = parse(
+            r#"
+        from orders
+        select [customer_no, gross, tax, gross - tax]
+        take 20
+        "#,
+        )?;
+
+        let query2 = parse(
+            r#"
+        from table_1
+        join customers [customer_no]
+        "#,
+        )?;
+
+        let (res1, context) = resolve(query1.nodes, None)?;
+        let (res2, context) = resolve(query2.nodes, Some(context))?;
+
+        let (mat, frame, context) = materialize(find_pipeline(res1), context.into(), None)?;
+
+        assert_yaml_snapshot!(mat.functions, @r###"
+        ---
+        - Transform:
+            From:
+              name: orders
+              alias: ~
+              declared_at: 30
+        - Transform:
+            Select:
+              assigns:
+                - Ident: customer_no
+                - Ident: gross
+                - Ident: tax
+                - Expr:
+                    - Ident: gross
+                    - Raw: "-"
+                    - Ident: tax
+              group: []
+              window: ~
+              sort: ~
+        - Transform:
+            Take: 20
+        "###);
+        assert_yaml_snapshot!(frame.columns, @r###"
+        ---
+        - Ident: customer_no
+        - Ident: gross
+        - Ident: tax
+        - Expr:
+            - Ident: gross
+            - Raw: "-"
+            - Ident: tax
+        "###);
+
+        let (mat, frame, _) = materialize(find_pipeline(res2), context, None)?;
+
+        assert_yaml_snapshot!(mat.functions, @r###"
+        ---
+        - Transform:
+            From:
+              name: table_1
+              alias: ~
+              declared_at: 35
+        - Transform:
+            Join:
+              side: Inner
+              with:
+                name: customers
+                alias: ~
+                declared_at: 36
+              filter:
+                Using:
+                  - Ident: customer_no
+        "###);
+        assert_yaml_snapshot!(frame.columns, @r###"
+        ---
+        - Ident: table_1.*
+        - Ident: customers.*
+        - Ident: customer_no
         "###);
 
         Ok(())
diff --git a/prql-compiler/src/sql/translator.rs b/prql-compiler/src/sql/translator.rs
index cb9093c7..769d5356 100644
--- a/prql-compiler/src/sql/translator.rs
+++ b/prql-compiler/src/sql/translator.rs
@@ -24,6 +24,7 @@ use crate::ast::*;
 use crate::error::{Error, Reason};
 use crate::semantic::Context;
 
+use super::materializer::MaterializationContext;
 use super::{un_group, MaterializedFrame};
 
 /// Translate a PRQL AST into a SQL string.
@@ -40,26 +41,21 @@ pub fn translate(query: Query, context: Context) -> Result<String> {
     Ok(formatted)
 }
 
-pub fn translate_query(query: Query, mut context: Context) -> Result<sql_ast::Query> {
+pub fn translate_query(query: Query, context: Context) -> Result<sql_ast::Query> {
     // extract tables and the pipeline
     let tables = into_tables(query.nodes)?;
 
+    let mut context = MaterializationContext::from(context);
+
     // split to atomics
-    let atomics = atomic_tables_of_tables(tables)?;
+    let atomics = atomic_tables_of_tables(tables, &mut context)?;
 
     // materialize each atomic in two stages
     let mut materialized = Vec::new();
     for t in atomics {
-        let last_node = t.pipeline.functions.last().unwrap();
-        let frame = last_node.frame.clone().unwrap();
+        let table_id = t.name.clone().and_then(|x| x.declared_at);
 
-        let rename_to = if t.name.is_empty() {
-            None
-        } else {
-            Some(t.name.as_str())
-        };
-
-        let (pipeline, frame, c) = super::materialize(t.pipeline, frame, context, rename_to)?;
+        let (pipeline, frame, c) = super::materialize(t.pipeline, context, table_id)?;
         context = c;
 
         materialized.push(AtomicTable {
@@ -97,7 +93,7 @@ pub fn translate_query(query: Query, mut context: Context) -> Result<sql_ast::Qu
 }
 
 pub struct AtomicTable {
-    name: String,
+    name: Option<TableRef>,
     pipeline: Pipeline,
     frame: Option<MaterializedFrame>,
 }
@@ -118,7 +114,7 @@ fn into_tables(nodes: Vec<Node>) -> Result<Vec<Table>> {
 
 fn table_to_sql_cte(table: AtomicTable, dialect: &Dialect) -> Result<sql_ast::Cte> {
     let alias = sql_ast::TableAlias {
-        name: Item::Ident(table.name.clone()).try_into()?,
+        name: Item::Ident(table.name.clone().unwrap().name).try_into()?,
         columns: vec![],
     };
     Ok(sql_ast::Cte {
@@ -272,7 +268,7 @@ fn sql_query_of_atomic_table(table: AtomicTable, dialect: &Dialect) -> Result<sq
         None => vec![],
         _ => unreachable!("Expected an aggregate transformation"),
     };
-    let group_by = Node::into_list_of_nodes(group_bys).item.try_into()?;
+    let group_by = Item::List(group_bys).try_into()?;
 
     let dialect = dialect.handler();
 
@@ -363,12 +359,15 @@ fn atomic_pipelines_of_pipeline(pipeline: Pipeline) -> Result<Vec<AtomicTable>>
 
 /// Converts a series of tables into a series of atomic tables, by putting the
 /// next pipeline's `from` as the current pipelines's table name.
-fn atomic_tables_of_tables(tables: Vec<Table>) -> Result<Vec<AtomicTable>> {
+fn atomic_tables_of_tables(
+    tables: Vec<Table>,
+    context: &mut MaterializationContext,
+) -> Result<Vec<AtomicTable>> {
     let mut atomics = Vec::new();
     let mut index = 0;
-    for t in tables {
+    for table in tables {
         // split table into atomics
-        let pipeline = t.pipeline.item.into_pipeline()?;
+        let pipeline = table.pipeline.item.into_pipeline()?;
         let mut t_atomics: Vec<_> = atomic_pipelines_of_pipeline(pipeline)?;
 
         let (last, ctes) = t_atomics
@@ -378,28 +377,37 @@ fn atomic_tables_of_tables(tables: Vec<Table>) -> Result<Vec<AtomicTable>> {
         // generate table names for all but last table
         let mut last_name = None;
         for cte in ctes {
-            prepend_with_from(&mut cte.pipeline, &last_name);
+            prepend_with_from(&mut cte.pipeline, last_name);
+
+            let name = format!("table_{index}");
+            let id = context.declare_table(&name);
 
-            cte.name = format!("table_{index}");
+            cte.name = Some(TableRef {
+                name,
+                alias: None,
+                declared_at: Some(id),
+            });
             index += 1;
-            last_name = Some(cte.name.clone());
+
+            last_name = cte.name.clone();
         }
 
         // use original table name
-        prepend_with_from(&mut last.pipeline, &last_name);
-        last.name = t.name;
+        prepend_with_from(&mut last.pipeline, last_name);
+        last.name = Some(TableRef {
+            name: table.name,
+            alias: None,
+            declared_at: table.id,
+        });
 
         atomics.extend(t_atomics);
     }
     Ok(atomics)
 }
 
-fn prepend_with_from(pipeline: &mut Pipeline, last_name: &Option<String>) {
-    if let Some(last_name) = last_name {
-        let from = Transform::From(TableRef {
-            name: last_name.clone(),
-            alias: None,
-        });
+fn prepend_with_from(pipeline: &mut Pipeline, table: Option<TableRef>) {
+    if let Some(table) = table {
+        let from = Transform::From(table);
         pipeline.functions.insert(0, Item::Transform(from).into());
     }
 }
@@ -425,7 +433,7 @@ impl TryFrom<Item> for SelectItem {
                 alias: sql_ast::Ident::new(named.name),
                 expr: named.expr.item.try_into()?,
             },
-            _ => bail!("Can't convert to SelectItem at the moment; {:?}", item),
+            _ => bail!("Can't convert to SelectItem; {:?}", item),
         })
     }
 }
@@ -535,7 +543,7 @@ impl TryFrom<Item> for Expr {
                     fractional_seconds_precision: None,
                 })
             }
-            _ => bail!("Can't convert to Expr at the moment; {item:?}"),
+            _ => bail!("Can't convert to Expr; {item:?}"),
         })
     }
 }
@@ -543,15 +551,8 @@ impl TryFrom<Item> for Vec<Expr> {
     type Error = anyhow::Error;
     fn try_from(item: Item) -> Result<Self> {
         match item {
-            Item::List(_) => Ok(Into::<Node>::into(item)
-                // TODO: implement for non-single item ListItems
-                .into_inner_list_nodes()?
-                .into_iter()
-                .map(|x| x.item.try_into())
-                .try_collect()?),
-            _ => Err(anyhow!(
-                "Can't convert to Vec<Expr> at the moment; {item:?}"
-            )),
+            Item::List(nodes) => Ok(nodes.into_iter().map(|x| x.item.try_into()).try_collect()?),
+            _ => Err(anyhow!("Can't convert to Vec<Expr>; {item:?}")),
         }
     }
 }
@@ -561,13 +562,14 @@ impl TryFrom<Item> for sql_ast::Ident {
         match item {
             Item::Ident(ident) => Ok(sql_ast::Ident::new(ident)),
             Item::Raw(ident) => Ok(sql_ast::Ident::new(ident)),
-            _ => Err(anyhow!("Can't convert to Ident at the moment; {item:?}")),
+            _ => Err(anyhow!("Can't convert to Ident; {item:?}")),
         }
     }
 }
 impl From<Vec<Node>> for Table {
     fn from(functions: Vec<Node>) -> Self {
         Table {
+            id: None,
             name: String::default(),
             pipeline: Box::new(Item::Pipeline(functions.into()).into()),
         }
@@ -576,7 +578,7 @@ impl From<Vec<Node>> for Table {
 impl From<Vec<Node>> for AtomicTable {
     fn from(functions: Vec<Node>) -> Self {
         AtomicTable {
-            name: String::default(),
+            name: None,
             pipeline: functions.into(),
             frame: None,
         }
@@ -651,8 +653,8 @@ SString:
     #[test]
     fn test_try_from_list_to_vec_expr() -> Result<()> {
         let item = Item::List(vec![
-            ListItem(Item::Ident("a".to_owned()).into()),
-            ListItem(Item::Ident("b".to_owned()).into()),
+            Item::Ident("a".to_owned()).into(),
+            Item::Ident("b".to_owned()).into(),
         ]);
         let expr: Vec<Expr> = item.try_into()?;
         assert_debug_snapshot!(expr, @r###"
@@ -880,7 +882,7 @@ take 20
         from newest_employees
         join average_salaries [country]
         select [name, salary, average_country_salary]
-    "#,
+        "#,
         )?;
         let sql = resolve_and_translate(query)?;
         assert_display_snapshot!(sql,
@@ -905,7 +907,7 @@ take 20
         )
         SELECT
           name,
-          salary,
+          average_salaries.salary,
           average_salaries.average_country_salary
         FROM
           newest_employees


Test patch:
diff --git a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_2.snap b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_2.snap
index a247ffab..4d3f490a 100644
--- a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_2.snap
+++ b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_2.snap
@@ -2,146 +2,144 @@
 source: prql-compiler/src/sql/materializer.rs
 expression: mat
 ---
-- Pipeline:
-    value: ~
-    functions:
-      - Transform:
-          From:
-            name: employees
-            alias: ~
-      - Transform:
-          Filter:
+- Transform:
+    From:
+      name: employees
+      alias: ~
+      declared_at: 35
+- Transform:
+    Filter:
+      - Expr:
+          - Ident: country
+          - Raw: "="
+          - String: USA
+- Transform:
+    Derive:
+      assigns:
+        - Expr:
+            - Ident: salary
+            - Raw: +
+            - Ident: payroll_tax
+        - Expr:
             - Expr:
-                - Ident: country
-                - Raw: "="
-                - String: USA
-      - Transform:
-          Derive:
-            assigns:
+                - Ident: salary
+                - Raw: +
+                - Ident: payroll_tax
+            - Raw: +
+            - Ident: benefits_cost
+      group: []
+      window: ~
+      sort: ~
+- Transform:
+    Filter:
+      - Expr:
+          - Expr:
               - Expr:
                   - Ident: salary
                   - Raw: +
                   - Ident: payroll_tax
-              - Expr:
+              - Raw: +
+              - Ident: benefits_cost
+          - Raw: ">"
+          - Raw: "0"
+- Transform:
+    Group:
+      by:
+        - Ident: title
+        - Ident: country
+      pipeline:
+        Pipeline:
+          value: ~
+          functions:
+            - Transform:
+                Aggregate:
+                  assigns:
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Ident: salary
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Ident: salary
+                        - String: )
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Expr:
+                              - Ident: salary
+                              - Raw: +
+                              - Ident: payroll_tax
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Expr:
+                              - Ident: salary
+                              - Raw: +
+                              - Ident: payroll_tax
+                        - String: )
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Expr:
+                              - Expr:
+                                  - Ident: salary
+                                  - Raw: +
+                                  - Ident: payroll_tax
+                              - Raw: +
+                              - Ident: benefits_cost
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Expr:
+                              - Expr:
+                                  - Ident: salary
+                                  - Raw: +
+                                  - Ident: payroll_tax
+                              - Raw: +
+                              - Ident: benefits_cost
+                        - String: )
+                    - SString:
+                        - String: COUNT(*)
+                  group:
+                    - Ident: title
+                    - Ident: country
+                  window: ~
+                  sort: ~
+- Transform:
+    Sort:
+      - direction: Asc
+        column:
+          SString:
+            - String: SUM(
+            - Expr:
+                Expr:
                   - Expr:
                       - Ident: salary
                       - Raw: +
                       - Ident: payroll_tax
                   - Raw: +
                   - Ident: benefits_cost
-            group: []
-            window: ~
-            sort: ~
-      - Transform:
-          Filter:
-            - Expr:
-                - Expr:
+            - String: )
+- Transform:
+    Filter:
+      - Expr:
+          - SString:
+              - String: SUM(
+              - Expr:
+                  Expr:
                     - Expr:
                         - Ident: salary
                         - Raw: +
                         - Ident: payroll_tax
                     - Raw: +
                     - Ident: benefits_cost
-                - Raw: ">"
-                - Raw: "0"
-      - Transform:
-          Group:
-            by:
-              - Ident: title
-              - Ident: country
-            pipeline:
-              Pipeline:
-                value: ~
-                functions:
-                  - Transform:
-                      Aggregate:
-                        assigns:
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Ident: salary
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Ident: salary
-                              - String: )
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Expr:
-                                    - Ident: salary
-                                    - Raw: +
-                                    - Ident: payroll_tax
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Expr:
-                                    - Ident: salary
-                                    - Raw: +
-                                    - Ident: payroll_tax
-                              - String: )
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Expr:
-                                    - Expr:
-                                        - Ident: salary
-                                        - Raw: +
-                                        - Ident: payroll_tax
-                                    - Raw: +
-                                    - Ident: benefits_cost
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Expr:
-                                    - Expr:
-                                        - Ident: salary
-                                        - Raw: +
-                                        - Ident: payroll_tax
-                                    - Raw: +
-                                    - Ident: benefits_cost
-                              - String: )
-                          - SString:
-                              - String: COUNT(*)
-                        group:
-                          - Ident: title
-                          - Ident: country
-                        window: ~
-                        sort: ~
-      - Transform:
-          Sort:
-            - direction: Asc
-              column:
-                SString:
-                  - String: SUM(
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: salary
-                            - Raw: +
-                            - Ident: payroll_tax
-                        - Raw: +
-                        - Ident: benefits_cost
-                  - String: )
-      - Transform:
-          Filter:
-            - Expr:
-                - SString:
-                    - String: SUM(
-                    - Expr:
-                        Expr:
-                          - Expr:
-                              - Ident: salary
-                              - Raw: +
-                              - Ident: payroll_tax
-                          - Raw: +
-                          - Ident: benefits_cost
-                    - String: )
-                - Raw: ">"
-                - Raw: "200"
-      - Transform:
-          Take: 20
+              - String: )
+          - Raw: ">"
+          - Raw: "200"
+- Transform:
+    Take: 20
 
diff --git a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_3.snap b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_3.snap
index ab006653..a7b82008 100644
--- a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_3.snap
+++ b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__materialize_3.snap
@@ -2,185 +2,183 @@
 source: prql-compiler/src/sql/materializer.rs
 expression: mat
 ---
-- Pipeline:
-    value: ~
-    functions:
-      - Transform:
-          From:
-            name: prices
-            alias: ~
-      - Transform:
-          Derive:
-            assigns:
-              - SString:
-                  - String: "IF(is_valid_price, "
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: prices_adj
-                            - Raw: /
-                            - SString:
-                                - String: lag_day_todo(
-                                - Expr:
-                                    Ident: prices_adj
-                                - String: )
-                        - Raw: "-"
-                        - Raw: "1"
-                        - Raw: +
-                        - Ident: div_ret
-                  - String: ", NULL)"
-              - SString:
-                  - String: "IF(is_valid_price, "
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: prices_usd
-                            - Raw: /
-                            - SString:
-                                - String: lag_day_todo(
-                                - Expr:
-                                    Ident: prices_usd
-                                - String: )
-                        - Raw: "-"
-                        - Raw: "1"
-                        - Raw: +
-                        - Ident: div_ret
-                  - String: ", NULL)"
-              - Expr:
+- Transform:
+    From:
+      name: prices
+      alias: ~
+      declared_at: 40
+- Transform:
+    Derive:
+      assigns:
+        - SString:
+            - String: "IF(is_valid_price, "
+            - Expr:
+                Expr:
                   - Expr:
+                      - Ident: prices_adj
+                      - Raw: /
                       - SString:
-                          - String: "IF(is_valid_price, "
+                          - String: lag_day_todo(
                           - Expr:
-                              Expr:
-                                - Expr:
-                                    - Ident: prices_adj
-                                    - Raw: /
-                                    - SString:
-                                        - String: lag_day_todo(
-                                        - Expr:
-                                            Ident: prices_adj
-                                        - String: )
-                                - Raw: "-"
-                                - Raw: "1"
-                                - Raw: +
-                                - Ident: div_ret
-                          - String: ", NULL)"
-                      - Raw: "-"
-                      - Raw: "0.2"
-                  - Raw: /
-                  - Raw: "252"
-              - Expr:
+                              Ident: prices_adj
+                          - String: )
+                  - Raw: "-"
+                  - Raw: "1"
+                  - Raw: +
+                  - Ident: div_ret
+            - String: ", NULL)"
+        - SString:
+            - String: "IF(is_valid_price, "
+            - Expr:
+                Expr:
                   - Expr:
+                      - Ident: prices_usd
+                      - Raw: /
                       - SString:
-                          - String: "IF(is_valid_price, "
+                          - String: lag_day_todo(
                           - Expr:
-                              Expr:
-                                - Expr:
-                                    - Ident: prices_usd
-                                    - Raw: /
-                                    - SString:
-                                        - String: lag_day_todo(
-                                        - Expr:
-                                            Ident: prices_usd
-                                        - String: )
-                                - Raw: "-"
-                                - Raw: "1"
-                                - Raw: +
-                                - Ident: div_ret
-                          - String: ", NULL)"
-                      - Raw: "-"
-                      - Raw: "0.2"
-                  - Raw: /
-                  - Raw: "252"
-            group: []
-            window: ~
-            sort: ~
-      - Transform:
-          Select:
-            assigns:
-              - Ident: date
-              - Ident: sec_id
-              - SString:
-                  - String: "IF(is_valid_price, "
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: prices_adj
-                            - Raw: /
-                            - SString:
-                                - String: lag_day_todo(
-                                - Expr:
-                                    Ident: prices_adj
-                                - String: )
-                        - Raw: "-"
-                        - Raw: "1"
-                        - Raw: +
-                        - Ident: div_ret
-                  - String: ", NULL)"
-              - SString:
-                  - String: "IF(is_valid_price, "
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: prices_usd
-                            - Raw: /
-                            - SString:
-                                - String: lag_day_todo(
-                                - Expr:
-                                    Ident: prices_usd
-                                - String: )
-                        - Raw: "-"
-                        - Raw: "1"
-                        - Raw: +
-                        - Ident: div_ret
-                  - String: ", NULL)"
-              - Expr:
+                              Ident: prices_usd
+                          - String: )
+                  - Raw: "-"
+                  - Raw: "1"
+                  - Raw: +
+                  - Ident: div_ret
+            - String: ", NULL)"
+        - Expr:
+            - Expr:
+                - SString:
+                    - String: "IF(is_valid_price, "
+                    - Expr:
+                        Expr:
+                          - Expr:
+                              - Ident: prices_adj
+                              - Raw: /
+                              - SString:
+                                  - String: lag_day_todo(
+                                  - Expr:
+                                      Ident: prices_adj
+                                  - String: )
+                          - Raw: "-"
+                          - Raw: "1"
+                          - Raw: +
+                          - Ident: div_ret
+                    - String: ", NULL)"
+                - Raw: "-"
+                - Raw: "0.2"
+            - Raw: /
+            - Raw: "252"
+        - Expr:
+            - Expr:
+                - SString:
+                    - String: "IF(is_valid_price, "
+                    - Expr:
+                        Expr:
+                          - Expr:
+                              - Ident: prices_usd
+                              - Raw: /
+                              - SString:
+                                  - String: lag_day_todo(
+                                  - Expr:
+                                      Ident: prices_usd
+                                  - String: )
+                          - Raw: "-"
+                          - Raw: "1"
+                          - Raw: +
+                          - Ident: div_ret
+                    - String: ", NULL)"
+                - Raw: "-"
+                - Raw: "0.2"
+            - Raw: /
+            - Raw: "252"
+      group: []
+      window: ~
+      sort: ~
+- Transform:
+    Select:
+      assigns:
+        - Ident: date
+        - Ident: sec_id
+        - SString:
+            - String: "IF(is_valid_price, "
+            - Expr:
+                Expr:
                   - Expr:
+                      - Ident: prices_adj
+                      - Raw: /
                       - SString:
-                          - String: "IF(is_valid_price, "
+                          - String: lag_day_todo(
                           - Expr:
-                              Expr:
-                                - Expr:
-                                    - Ident: prices_adj
-                                    - Raw: /
-                                    - SString:
-                                        - String: lag_day_todo(
-                                        - Expr:
-                                            Ident: prices_adj
-                                        - String: )
-                                - Raw: "-"
-                                - Raw: "1"
-                                - Raw: +
-                                - Ident: div_ret
-                          - String: ", NULL)"
-                      - Raw: "-"
-                      - Raw: "0.2"
-                  - Raw: /
-                  - Raw: "252"
-              - Expr:
+                              Ident: prices_adj
+                          - String: )
+                  - Raw: "-"
+                  - Raw: "1"
+                  - Raw: +
+                  - Ident: div_ret
+            - String: ", NULL)"
+        - SString:
+            - String: "IF(is_valid_price, "
+            - Expr:
+                Expr:
                   - Expr:
+                      - Ident: prices_usd
+                      - Raw: /
                       - SString:
-                          - String: "IF(is_valid_price, "
+                          - String: lag_day_todo(
+                          - Expr:
+                              Ident: prices_usd
+                          - String: )
+                  - Raw: "-"
+                  - Raw: "1"
+                  - Raw: +
+                  - Ident: div_ret
+            - String: ", NULL)"
+        - Expr:
+            - Expr:
+                - SString:
+                    - String: "IF(is_valid_price, "
+                    - Expr:
+                        Expr:
+                          - Expr:
+                              - Ident: prices_adj
+                              - Raw: /
+                              - SString:
+                                  - String: lag_day_todo(
+                                  - Expr:
+                                      Ident: prices_adj
+                                  - String: )
+                          - Raw: "-"
+                          - Raw: "1"
+                          - Raw: +
+                          - Ident: div_ret
+                    - String: ", NULL)"
+                - Raw: "-"
+                - Raw: "0.2"
+            - Raw: /
+            - Raw: "252"
+        - Expr:
+            - Expr:
+                - SString:
+                    - String: "IF(is_valid_price, "
+                    - Expr:
+                        Expr:
                           - Expr:
-                              Expr:
-                                - Expr:
-                                    - Ident: prices_usd
-                                    - Raw: /
-                                    - SString:
-                                        - String: lag_day_todo(
-                                        - Expr:
-                                            Ident: prices_usd
-                                        - String: )
-                                - Raw: "-"
-                                - Raw: "1"
-                                - Raw: +
-                                - Ident: div_ret
-                          - String: ", NULL)"
-                      - Raw: "-"
-                      - Raw: "0.2"
-                  - Raw: /
-                  - Raw: "252"
-            group: []
-            window: ~
-            sort: ~
+                              - Ident: prices_usd
+                              - Raw: /
+                              - SString:
+                                  - String: lag_day_todo(
+                                  - Expr:
+                                      Ident: prices_usd
+                                  - String: )
+                          - Raw: "-"
+                          - Raw: "1"
+                          - Raw: +
+                          - Ident: div_ret
+                    - String: ", NULL)"
+                - Raw: "-"
+                - Raw: "0.2"
+            - Raw: /
+            - Raw: "252"
+      group: []
+      window: ~
+      sort: ~
 
diff --git a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__replace_variables_2.snap b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__replace_variables_2.snap
index e8d07e3c..281b651f 100644
--- a/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__replace_variables_2.snap
+++ b/prql-compiler/src/sql/snapshots/prql_compiler__sql__materializer__test__replace_variables_2.snap
@@ -2,146 +2,144 @@
 source: prql-compiler/src/sql/materializer.rs
 expression: "&mat"
 ---
-- Pipeline:
-    value: ~
-    functions:
-      - Transform:
-          From:
-            name: employees
-            alias: ~
-      - Transform:
-          Filter:
+- Transform:
+    From:
+      name: employees
+      alias: ~
+      declared_at: 35
+- Transform:
+    Filter:
+      - Expr:
+          - Ident: country
+          - Raw: "="
+          - String: USA
+- Transform:
+    Derive:
+      assigns:
+        - Expr:
+            - Ident: salary
+            - Raw: +
+            - Ident: payroll_tax
+        - Expr:
             - Expr:
-                - Ident: country
-                - Raw: "="
-                - String: USA
-      - Transform:
-          Derive:
-            assigns:
+                - Ident: salary
+                - Raw: +
+                - Ident: payroll_tax
+            - Raw: +
+            - Ident: benefits_cost
+      group: []
+      window: ~
+      sort: ~
+- Transform:
+    Filter:
+      - Expr:
+          - Expr:
               - Expr:
                   - Ident: salary
                   - Raw: +
                   - Ident: payroll_tax
-              - Expr:
+              - Raw: +
+              - Ident: benefits_cost
+          - Raw: ">"
+          - Raw: "0"
+- Transform:
+    Group:
+      by:
+        - Ident: title
+        - Ident: country
+      pipeline:
+        Pipeline:
+          value: ~
+          functions:
+            - Transform:
+                Aggregate:
+                  assigns:
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Ident: salary
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Ident: salary
+                        - String: )
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Expr:
+                              - Ident: salary
+                              - Raw: +
+                              - Ident: payroll_tax
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Expr:
+                              - Ident: salary
+                              - Raw: +
+                              - Ident: payroll_tax
+                        - String: )
+                    - SString:
+                        - String: AVG(
+                        - Expr:
+                            Expr:
+                              - Expr:
+                                  - Ident: salary
+                                  - Raw: +
+                                  - Ident: payroll_tax
+                              - Raw: +
+                              - Ident: benefits_cost
+                        - String: )
+                    - SString:
+                        - String: SUM(
+                        - Expr:
+                            Expr:
+                              - Expr:
+                                  - Ident: salary
+                                  - Raw: +
+                                  - Ident: payroll_tax
+                              - Raw: +
+                              - Ident: benefits_cost
+                        - String: )
+                    - SString:
+                        - String: COUNT(*)
+                  group:
+                    - Ident: title
+                    - Ident: country
+                  window: ~
+                  sort: ~
+- Transform:
+    Sort:
+      - direction: Asc
+        column:
+          SString:
+            - String: SUM(
+            - Expr:
+                Expr:
                   - Expr:
                       - Ident: salary
                       - Raw: +
                       - Ident: payroll_tax
                   - Raw: +
                   - Ident: benefits_cost
-            group: []
-            window: ~
-            sort: ~
-      - Transform:
-          Filter:
-            - Expr:
-                - Expr:
+            - String: )
+- Transform:
+    Filter:
+      - Expr:
+          - SString:
+              - String: SUM(
+              - Expr:
+                  Expr:
                     - Expr:
                         - Ident: salary
                         - Raw: +
                         - Ident: payroll_tax
                     - Raw: +
                     - Ident: benefits_cost
-                - Raw: ">"
-                - Raw: "0"
-      - Transform:
-          Group:
-            by:
-              - Ident: title
-              - Ident: country
-            pipeline:
-              Pipeline:
-                value: ~
-                functions:
-                  - Transform:
-                      Aggregate:
-                        assigns:
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Ident: salary
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Ident: salary
-                              - String: )
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Expr:
-                                    - Ident: salary
-                                    - Raw: +
-                                    - Ident: payroll_tax
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Expr:
-                                    - Ident: salary
-                                    - Raw: +
-                                    - Ident: payroll_tax
-                              - String: )
-                          - SString:
-                              - String: AVG(
-                              - Expr:
-                                  Expr:
-                                    - Expr:
-                                        - Ident: salary
-                                        - Raw: +
-                                        - Ident: payroll_tax
-                                    - Raw: +
-                                    - Ident: benefits_cost
-                              - String: )
-                          - SString:
-                              - String: SUM(
-                              - Expr:
-                                  Expr:
-                                    - Expr:
-                                        - Ident: salary
-                                        - Raw: +
-                                        - Ident: payroll_tax
-                                    - Raw: +
-                                    - Ident: benefits_cost
-                              - String: )
-                          - SString:
-                              - String: COUNT(*)
-                        group:
-                          - Ident: title
-                          - Ident: country
-                        window: ~
-                        sort: ~
-      - Transform:
-          Sort:
-            - direction: Asc
-              column:
-                SString:
-                  - String: SUM(
-                  - Expr:
-                      Expr:
-                        - Expr:
-                            - Ident: salary
-                            - Raw: +
-                            - Ident: payroll_tax
-                        - Raw: +
-                        - Ident: benefits_cost
-                  - String: )
-      - Transform:
-          Filter:
-            - Expr:
-                - SString:
-                    - String: SUM(
-                    - Expr:
-                        Expr:
-                          - Expr:
-                              - Ident: salary
-                              - Raw: +
-                              - Ident: payroll_tax
-                          - Raw: +
-                          - Ident: benefits_cost
-                    - String: )
-                - Raw: ">"
-                - Raw: "200"
-      - Transform:
-          Take: 20
+              - String: )
+          - Raw: ">"
+          - Raw: "200"
+- Transform:
+    Take: 20
 
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@cte-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@cte-0.prql.snap
index 962223c0..51a4a098 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@cte-0.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@cte-0.prql.snap
@@ -23,7 +23,7 @@ WITH newest_employees AS (
 )
 SELECT
   name,
-  salary,
+  average_salaries.salary,
   average_salaries.average_country_salary
 FROM
   newest_employees
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@employees-0-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@employees-0-0.prql.snap
index 8c72a075..bd54ab15 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@employees-0-0.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@employees-0-0.prql.snap
@@ -27,7 +27,7 @@ table_1 AS (
 )
 SELECT
   departments.dept_name,
-  t.title,
+  table_1.title,
   table_1.avg_salary
 FROM
   table_1
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@employees-1-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@employees-1-0.prql.snap
index fc28c785..cce958c8 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@employees-1-0.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@employees-1-0.prql.snap
@@ -30,7 +30,7 @@ table_1 AS (
 )
 SELECT
   departments.dept_name,
-  de.gender,
+  table_1.gender,
   table_1.salary_avg,
   table_1.salary_sd
 FROM
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@employees-3-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@employees-3-0.prql.snap
index 940dafc3..4ba3342e 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@employees-3-0.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@employees-3-0.prql.snap
@@ -17,7 +17,7 @@ WITH table_0 AS (
     dept_emp.dept_no
 )
 SELECT
-  dept_emp.dept_no,
+  table_0.dept_no,
   table_0.salary,
   employees.gender,
   titles.title
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@introduction-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@introduction-0.prql.snap
index 26483724..dd0fe54e 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@introduction-0.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@introduction-0.prql.snap
@@ -30,14 +30,7 @@ WITH table_0 AS (
     20
 )
 SELECT
-  title,
-  AVG(salary),
-  AVG(salary + payroll_tax),
-  SUM(salary),
-  SUM(salary + payroll_tax),
-  AVG(salary + payroll_tax + benefits_cost),
-  sum_gross_cost,
-  table_0.ct,
+  table_0.*,
   countries.*,
   country_code,
   countries.true AS always_true,
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-2.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-2.prql.snap
index 266d4531..030834db 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-2.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-2.prql.snap
@@ -6,6 +6,6 @@ input_file: reference/tests/examples/examples/list-equivalence-2.prql
 SELECT
   employees.*,
   salary + payroll_tax AS gross_salary,
-  salary + payroll_tax + benefits_cost AS gross_cost
+  gross_salary + benefits_cost AS gross_cost
 FROM
   employees
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-3.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-3.prql.snap
index c57bddbd..b301494f 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-3.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@list-equivalence-3.prql.snap
@@ -6,6 +6,6 @@ input_file: reference/tests/examples/examples/list-equivalence-3.prql
 SELECT
   employees.*,
   salary + payroll_tax AS gross_salary,
-  salary + payroll_tax + benefits_cost AS gross_cost
+  gross_salary + benefits_cost AS gross_cost
 FROM
   employees
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@transforms-3.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@transforms-3.prql.snap
index 4102297c..b68c34db 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@transforms-3.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@transforms-3.prql.snap
@@ -6,6 +6,6 @@ input_file: reference/tests/examples/transforms-3.prql
 SELECT
   employees.*,
   salary + payroll_tax AS gross_salary,
-  salary + payroll_tax + benefits_cost AS gross_cost
+  gross_salary + benefits_cost AS gross_cost
 FROM
   employees
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@variables-1.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@variables-1.prql.snap
index 2b627a2f..18119087 100644
--- a/reference/tests/snapshots/snapshot__run_reference_examples@variables-1.prql.snap
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@variables-1.prql.snap
@@ -14,7 +14,7 @@ WITH table_0 AS (
 )
 SELECT
   AVG(table_0.emp_salary) / 1000 AS salary_k,
-  AVG(table_0.emp_salary) / 1000 * 1000 AS salary
+  salary_k * 1000 AS salary
 FROM
   table_0
   JOIN titles USING(emp_no)