Domain: dataflow query planner/optimizer

Issue:
Since we moved reporting onto the optimizing planner, some pipelines return wrong results: filters that combine with computed columns keep rows they should drop and drop rows they should keep — the surviving rows look as if the filter had been judged against the values from before the computation. One team hit a second shape of it with renamed columns, where a filter matched as though it were reading a completely different column than the one it names. Running the same pipelines with optimization turned off returns correct results every time, but that is not an acceptable fix: our plan dashboards and run-cost budgets depend on filters continuing to run ahead of the expensive computed-column and relabeling steps wherever that is sound. Results must match the unoptimized run exactly, and plans must keep pruning rows as early as they soundly can.

Repository files:
content: # Present so pytest adds the repository root to sys.path, making `import src`
# resolve from the test suite.

path: conftest.py

content: # Pure standard library unless the environment image pre-bakes wheels.

path: requirements.lock

path: src/__init__.py

content: """Plan cost estimation.

Estimates are unitless work numbers used by run-cost budgeting: each
stage charges a fixed per-row weight for every row entering it, and
passes an estimated row count downstream. The absolute numbers matter
less than the comparisons — the whole point of planning is that a plan
which prunes rows early estimates (and runs) cheaper than one that
does expensive per-row work first.

Row estimates use fixed selectivities so budgeting is deterministic: a
filter is assumed to keep half its input, an aggregation to produce a
tenth.
"""

from src.stages.aggregate import Aggregate
from src.stages.derive import Derive
from src.stages.filter import Filter
from src.stages.limit import Limit
from src.stages.rename import Rename
from src.stages.select import Select
from src.stages.sort import Sort

_WEIGHTS = (
    (Filter, 1),
    (Rename, 1),
    (Select, 1),
    (Sort, 2),
    (Derive, 4),
    (Aggregate, 5),
    (Limit, 0),
)


def stage_weight(stage):
    """Per-row work weight for ``stage``."""
    for stage_type, weight in _WEIGHTS:
        if isinstance(stage, stage_type):
            return weight
    return 1


def rows_leaving(stage, rows_entering):
    """Estimated row count leaving ``stage``."""
    if isinstance(stage, Filter):
        return rows_entering // 2
    if isinstance(stage, Limit):
        return min(stage.count, rows_entering)
    if isinstance(stage, Aggregate):
        return max(1, rows_entering // 10)
    return rows_entering


def estimate(stages, source_rows):
    """Total estimated work to run ``stages`` over ``source_rows`` rows."""
    total = 0
    rows = source_rows
    for stage in stages:
        total += rows * stage_weight(stage)
        rows = rows_leaving(stage, rows)
    return total


def breakdown(stages, source_rows):
    """Per-stage ``(describe, rows_entering, work)`` triples for reports."""
    out = []
    rows = source_rows
    for stage in stages:
        out.append((stage.describe(), rows, rows * stage_weight(stage)))
        rows = rows_leaving(stage, rows)
    return out

path: src/costs.py

content: """Loading rows from delimited text.

The library operates on plain lists of dicts; this module turns the
pipe-delimited exports our tools produce into that shape. The first
line names the columns; every following non-blank line is one row.
Values are strings unless a type is given for the column.
"""

from src.errors import PipelineError

_CASTS = {
    "str": str,
    "int": int,
    "float": float,
}


def rows_from_text(text, types=None, delimiter="|"):
    """Parse delimited ``text`` into a list of row dicts.

    ``types`` maps column names to ``"int"``, ``"float"`` or ``"str"``
    (the default). Cells are stripped of surrounding whitespace before
    casting, so aligned exports parse the same as compact ones.
    """
    types = types or {}
    for column in types:
        if types[column] not in _CASTS:
            raise PipelineError(
                "unknown type %r for column %s" % (types[column], column)
            )
    lines = [line for line in text.splitlines() if line.strip()]
    if not lines:
        raise PipelineError("no header line in input")
    columns = [cell.strip() for cell in lines[0].split(delimiter)]
    if len(set(columns)) != len(columns):
        raise PipelineError("duplicate column in header")
    rows = []
    for number, line in enumerate(lines[1:], start=2):
        cells = [cell.strip() for cell in line.split(delimiter)]
        if len(cells) != len(columns):
            raise PipelineError(
                "line %d has %d cells, expected %d"
                % (number, len(cells), len(columns))
            )
        row = {}
        for column, cell in zip(columns, cells):
            cast = _CASTS[types.get(column, "str")]
            try:
                row[column] = cast(cell)
            except ValueError:
                raise PipelineError(
                    "line %d: cannot cast %r to %s for column %s"
                    % (number, cell, types[column], column)
                )
        rows.append(row)
    return rows

path: src/datasource.py

content: """Plan execution.

The engine is deliberately simple: it materializes each step in order.
All cleverness about *which* order lives in the planner, and all
correctness checks live in validation — by the time a plan reaches the
engine it is assumed well-formed.
"""


def execute(stages, rows):
    """Run ``stages`` in order over ``rows`` and return the result rows."""
    current = [dict(row) for row in rows]
    for stage in stages:
        current = stage.apply(current)
    return current

path: src/engine.py

content: """Exception types for the pipeline library."""


class PipelineError(Exception):
    """Base class for all pipeline construction and execution errors."""


class UnknownColumn(PipelineError):
    """A stage or expression referenced a column that does not exist."""


class DuplicateColumn(PipelineError):
    """A stage would produce two columns with the same name."""


class InvalidStage(PipelineError):
    """A stage was constructed with invalid arguments."""

path: src/errors.py

content: """Expression trees for filters and computed columns.

Expressions are built from :func:`col` and :func:`lit` and combined with
Python operators::

    col("score") >= lit(50)
    (col("status") == lit("active")) & (col("score") * lit(2) > lit(10))

Comparisons and arithmetic accept plain Python values and wrap them in
:func:`lit` automatically. Boolean composition uses ``&``, ``|`` and ``~``
(``and``/``or``/``not`` cannot be overloaded in Python).

Every expression can be evaluated against a row (:func:`evaluate`),
asked which columns it mentions (:func:`references`), and rendered to a
stable human-readable string (:func:`render`) used by plan explains.
"""

from src.errors import InvalidStage, UnknownColumn

_FUNCTIONS = {
    "upper": lambda v: v.upper(),
    "lower": lambda v: v.lower(),
    "length": len,
    "abs": abs,
}


class Expr:
    """Base class for all expression nodes."""

    def __eq__(self, other):
        return Cmp("==", self, _wrap(other))

    def __ne__(self, other):
        return Cmp("!=", self, _wrap(other))

    def __lt__(self, other):
        return Cmp("<", self, _wrap(other))

    def __le__(self, other):
        return Cmp("<=", self, _wrap(other))

    def __gt__(self, other):
        return Cmp(">", self, _wrap(other))

    def __ge__(self, other):
        return Cmp(">=", self, _wrap(other))

    def __add__(self, other):
        return Arith("+", self, _wrap(other))

    def __sub__(self, other):
        return Arith("-", self, _wrap(other))

    def __mul__(self, other):
        return Arith("*", self, _wrap(other))

    def __truediv__(self, other):
        return Arith("/", self, _wrap(other))

    def __and__(self, other):
        return And(self, _wrap(other))

    def __or__(self, other):
        return Or(self, _wrap(other))

    def __invert__(self):
        return Not(self)

    # Expressions are structural values; identity-based hashing would be
    # misleading once __eq__ builds comparison nodes instead.
    __hash__ = None


class Lit(Expr):
    """A constant value."""

    def __init__(self, value):
        self.value = value


class Col(Expr):
    """A reference to a column by name."""

    def __init__(self, name):
        if not isinstance(name, str) or not name:
            raise InvalidStage("column names must be non-empty strings")
        self.name = name


class Arith(Expr):
    """Binary arithmetic: ``+``, ``-``, ``*`` or ``/``."""

    def __init__(self, op, left, right):
        self.op = op
        self.left = left
        self.right = right


class Cmp(Expr):
    """Binary comparison: ``==``, ``!=``, ``<``, ``<=``, ``>`` or ``>=``."""

    def __init__(self, op, left, right):
        self.op = op
        self.left = left
        self.right = right


class And(Expr):
    """Both sides truthy."""

    def __init__(self, left, right):
        self.left = left
        self.right = right


class Or(Expr):
    """Either side truthy."""

    def __init__(self, left, right):
        self.left = left
        self.right = right


class Not(Expr):
    """Logical negation."""

    def __init__(self, operand):
        self.operand = operand


class Func(Expr):
    """A built-in scalar function applied to one argument."""

    def __init__(self, name, operand):
        if name not in _FUNCTIONS:
            raise InvalidStage("unknown function: %s" % name)
        self.name = name
        self.operand = operand


def col(name):
    """Reference the column ``name``."""
    return Col(name)


def lit(value):
    """Wrap a constant ``value``."""
    return Lit(value)


def func(name, operand):
    """Apply the built-in function ``name`` to ``operand``."""
    return Func(name, _wrap(operand))


def _wrap(value):
    return value if isinstance(value, Expr) else Lit(value)


_ARITH = {
    "+": lambda a, b: a + b,
    "-": lambda a, b: a - b,
    "*": lambda a, b: a * b,
    "/": lambda a, b: a / b,
}

_CMP = {
    "==": lambda a, b: a == b,
    "!=": lambda a, b: a != b,
    "<": lambda a, b: a < b,
    "<=": lambda a, b: a <= b,
    ">": lambda a, b: a > b,
    ">=": lambda a, b: a >= b,
}


def evaluate(expression, row):
    """Evaluate ``expression`` against ``row`` (a mapping of column -> value)."""
    if isinstance(expression, Lit):
        return expression.value
    if isinstance(expression, Col):
        if expression.name not in row:
            raise UnknownColumn(expression.name)
        return row[expression.name]
    if isinstance(expression, Arith):
        return _ARITH[expression.op](
            evaluate(expression.left, row), evaluate(expression.right, row)
        )
    if isinstance(expression, Cmp):
        return _CMP[expression.op](
            evaluate(expression.left, row), evaluate(expression.right, row)
        )
    if isinstance(expression, And):
        return bool(evaluate(expression.left, row)) and bool(
            evaluate(expression.right, row)
        )
    if isinstance(expression, Or):
        return bool(evaluate(expression.left, row)) or bool(
            evaluate(expression.right, row)
        )
    if isinstance(expression, Not):
        return not evaluate(expression.operand, row)
    if isinstance(expression, Func):
        return _FUNCTIONS[expression.name](evaluate(expression.operand, row))
    raise InvalidStage("not an expression: %r" % (expression,))


def references(expression):
    """Return the set of column names ``expression`` mentions."""
    if isinstance(expression, Col):
        return frozenset((expression.name,))
    if isinstance(expression, (Arith, Cmp, And, Or)):
        return references(expression.left) | references(expression.right)
    if isinstance(expression, (Not, Func)):
        return references(expression.operand)
    return frozenset()


def render(expression):
    """Render ``expression`` as a stable, parseable string.

    The output is part of the plan-explain format, so it must not change
    between releases for the same expression tree.
    """
    if isinstance(expression, Lit):
        return repr(expression.value)
    if isinstance(expression, Col):
        return expression.name
    if isinstance(expression, (Arith, Cmp)):
        return "(%s %s %s)" % (
            render(expression.left),
            expression.op,
            render(expression.right),
        )
    if isinstance(expression, And):
        return "(%s and %s)" % (render(expression.left), render(expression.right))
    if isinstance(expression, Or):
        return "(%s or %s)" % (render(expression.left), render(expression.right))
    if isinstance(expression, Not):
        return "(not %s)" % render(expression.operand)
    if isinstance(expression, Func):
        return "%s(%s)" % (expression.name, render(expression.operand))
    raise InvalidStage("not an expression: %r" % (expression,))

path: src/expr.py

content: """Runtime work counters.

Where :mod:`src.costs` predicts how much work a plan will do, this
module records how much work it actually did: every stage reports the
number of rows entering it, bucketed by stage kind. Budget dashboards
compare the two to catch plans whose real cost drifts from the
estimate.

Counters accumulate across runs until :func:`reset` is called, so
callers measuring a single run should reset first.
"""

counters = {}


def observe(label, rows_entering):
    """Record that a stage of kind ``label`` processed ``rows_entering`` rows."""
    counters[label] = counters.get(label, 0) + rows_entering


def reset():
    """Clear all counters."""
    counters.clear()


def snapshot():
    """A stable copy of the counters for reporting."""
    return dict(sorted(counters.items()))

path: src/metrics.py

content: """The pipeline builder — the library's main entry point.

A pipeline is written as a fluent chain of steps and then either run
against rows or explained::

    from src.expr import col
    from src.pipeline import Pipeline

    p = (
        Pipeline()
        .filter(col("status") == "active")
        .derive("total", col("price") * col("quantity"))
        .sort("total", descending=True)
        .limit(10)
    )
    result = p.run(rows)
    for line in p.explain(["id", "status", "price", "quantity"]):
        print(line)

``run`` validates the pipeline, hands it to the planner for
optimization, and executes the optimized plan. Pass ``optimize=False``
to run the steps exactly as written — the results are the same either
way; only the cost changes.
"""

from src import engine, validate
from src.errors import PipelineError
from src.planner.optimizer import optimize as _optimize
from src.planner.plan import Plan
from src.stages.aggregate import Aggregate
from src.stages.derive import Derive
from src.stages.filter import Filter
from src.stages.limit import Limit
from src.stages.rename import Rename
from src.stages.select import Select
from src.stages.sort import Sort


class Pipeline:
    """A chain of stages built fluently and run as one unit."""

    def __init__(self):
        self._stages = []

    def filter(self, predicate):
        """Keep only rows matching ``predicate``."""
        self._stages.append(Filter(predicate))
        return self

    def derive(self, target, expression):
        """Compute ``expression`` into column ``target`` for every row."""
        self._stages.append(Derive(target, expression))
        return self

    def rename(self, mapping):
        """Rename columns per ``mapping`` (old name -> new name)."""
        self._stages.append(Rename(mapping))
        return self

    def select(self, *columns):
        """Keep only ``columns``, in the given order."""
        self._stages.append(Select(columns))
        return self

    def sort(self, *keys, descending=False):
        """Sort rows by ``keys``."""
        self._stages.append(Sort(keys, descending=descending))
        return self

    def limit(self, count):
        """Keep only the first ``count`` rows."""
        self._stages.append(Limit(count))
        return self

    def aggregate(self, keys, aggs):
        """Group by ``keys`` and compute the ``aggs`` columns."""
        self._stages.append(Aggregate(keys, aggs))
        return self

    def stages(self):
        """The steps of this pipeline, as written."""
        return list(self._stages)

    def run(self, rows, optimize=True):
        """Validate, plan, and execute this pipeline over ``rows``."""
        columns = _source_columns(rows)
        validate.check(self._stages, columns)
        stages = self._stages
        if optimize:
            stages = _optimize(stages, columns)
        return engine.execute(stages, rows)

    def explain(self, source_columns, optimize=True):
        """Render the plan that ``run`` would execute, one line per step."""
        columns = list(source_columns)
        validate.check(self._stages, columns)
        stages = self._stages
        if optimize:
            stages = _optimize(stages, columns)
        return Plan(columns, stages).lines()


def _source_columns(rows):
    if not rows:
        raise PipelineError("cannot infer source columns from zero rows")
    return list(rows[0])

path: src/pipeline.py

content: """Planning and optimization."""

from src.planner.plan import Plan

__all__ = ["Plan"]

path: src/planner/__init__.py

content: """Pipeline optimization.

The optimizer only performs rewrites that preserve results: for any
input, an optimized pipeline must return exactly the rows the
unoptimized pipeline returns. Within that constraint it pursues one
goal — do cheap row pruning as early as possible, so expensive per-row
work downstream touches fewer rows.

Two rewrites are applied today:

* **Filter pushdown** — each filter is moved as far toward the scan as
  the reordering rules allow.
* **Filter fusion** — adjacent filters are fused into a single pass so
  the row loop runs once instead of twice.
"""

from src.expr import And
from src.planner import rules
from src.stages.filter import Filter


def optimize(stages, source_columns):
    """Return an optimized arrangement of ``stages``."""
    stages = _push_filters(list(stages), source_columns)
    stages = _fuse_filters(stages)
    return stages


def _columns_entering(stages, source_columns):
    """For each stage, the set of columns present just upstream of it."""
    entering = [frozenset(source_columns)]
    for stage in stages[:-1]:
        entering.append(stage.output_columns(entering[-1]))
    return entering


def _push_filters(stages, source_columns):
    """Bubble each filter toward the scan while the rules allow it.

    Filters never move past other filters: their relative order is
    irrelevant to the result, and fusion cleans up any that end up
    adjacent.
    """
    moved = True
    while moved:
        moved = False
        entering = _columns_entering(stages, source_columns)
        for i in range(1, len(stages)):
            filt = stages[i]
            prev = stages[i - 1]
            if not isinstance(filt, Filter) or isinstance(prev, Filter):
                continue
            if rules.can_push_filter_below(filt, prev, entering[i - 1]):
                stages[i - 1], stages[i] = filt, prev
                moved = True
                break
    return stages


def _fuse_filters(stages):
    """Fuse runs of adjacent filters into one conjunctive filter."""
    fused = []
    for stage in stages:
        if (
            fused
            and isinstance(stage, Filter)
            and isinstance(fused[-1], Filter)
        ):
            fused[-1] = Filter(And(fused[-1].predicate, stage.predicate))
        else:
            fused.append(stage)
    return fused

path: src/planner/optimizer.py

content: """Plan representation.

A plan is the ordered list of stages the engine will actually run,
plus the source columns it scans. ``Plan.lines()`` is the explain
format: one stable line per step. Downstream tooling parses these
lines, so the format is part of the library's public surface.
"""


class Plan:
    """An executable, explainable arrangement of stages."""

    def __init__(self, source_columns, stages):
        self.source_columns = tuple(source_columns)
        self.stages = list(stages)

    def lines(self):
        """Render the plan, one line per step, scan first."""
        scan = "scan[%s]" % ", ".join(self.source_columns)
        return [scan] + [stage.describe() for stage in self.stages]

    def __repr__(self):
        return "<Plan %s>" % " | ".join(self.lines())

path: src/planner/plan.py

content: """Reordering rules used by the optimizer.

A rule is consulted every time the optimizer considers moving a filter
in front of the stage that currently precedes it. Rules must be
conservative: a move is only worth making when the reordered plan is
cheaper and returns the same rows.
"""

from src.expr import references
from src.stages.derive import Derive
from src.stages.rename import Rename
from src.stages.select import Select
from src.stages.sort import Sort

#: Stages that neither add nor remove rows. Filtering before one of
#: these keeps exactly the same set of rows in play, so each row's
#: filter verdict is unaffected by which side of the stage it runs on.
ROW_PRESERVING = (Derive, Rename, Select, Sort)


def can_push_filter_below(filt, stage, upstream_columns):
    """Return True when ``filt`` may run before ``stage``.

    A filter can only be evaluated at a point in the plan where every
    column it mentions is present. The move is therefore allowed when
    the preceding stage is row-preserving and all of the filter's
    columns are already available upstream of it.
    """
    if not isinstance(stage, ROW_PRESERVING):
        return False
    return references(filt.predicate) <= frozenset(upstream_columns)

path: src/planner/rules.py

content: """A small text form for pipelines.

Operations tooling stores saved queries as plain text, one step per
line, and compiles them to pipelines at run time::

    filter status == 'active'
    derive total = price * quantity
    rename total -> order_total
    sort order_total desc
    limit 25

Filters support a single comparison or comparisons joined with
``and``. Values are integers, floats, or single-quoted strings.
"""

from src.errors import PipelineError
from src.expr import Cmp, col, lit
from src.pipeline import Pipeline

_CMP_OPS = ("==", "!=", "<=", ">=", "<", ">")


def compile_text(text):
    """Compile query ``text`` into a :class:`Pipeline`."""
    pipeline = Pipeline()
    for number, raw in enumerate(text.splitlines(), start=1):
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        keyword, _, rest = line.partition(" ")
        handler = _HANDLERS.get(keyword)
        if handler is None:
            raise PipelineError("line %d: unknown step %r" % (number, keyword))
        try:
            handler(pipeline, rest.strip())
        except PipelineError as exc:
            raise PipelineError("line %d: %s" % (number, exc))
    return pipeline


def _parse_value(token):
    token = token.strip()
    if token.startswith("'") and token.endswith("'") and len(token) >= 2:
        return token[1:-1]
    try:
        return int(token)
    except ValueError:
        pass
    try:
        return float(token)
    except ValueError:
        raise PipelineError("cannot parse value %r" % token)


def _parse_comparison(clause):
    for op in _CMP_OPS:
        if op in clause:
            left, _, right = clause.partition(op)
            name = left.strip()
            if not name.isidentifier():
                raise PipelineError("bad column name %r" % name)
            return Cmp(op, col(name), lit(_parse_value(right)))
    raise PipelineError("no comparison in %r" % clause)


def _filter(pipeline, rest):
    clauses = rest.split(" and ")
    predicate = _parse_comparison(clauses[0])
    for clause in clauses[1:]:
        predicate = predicate & _parse_comparison(clause)
    pipeline.filter(predicate)


def _derive(pipeline, rest):
    target, eq, formula = rest.partition("=")
    target = target.strip()
    if not eq or not target.isidentifier():
        raise PipelineError("derive needs 'target = column <op> value-or-column'")
    formula = formula.strip()
    for op in ("+", "-", "*", "/"):
        if op in formula:
            left, _, right = formula.partition(op)
            left = left.strip()
            right = right.strip()
            if not left.isidentifier():
                raise PipelineError("bad column name %r" % left)
            left_expr = col(left)
            if right.isidentifier():
                right_expr = col(right)
            else:
                right_expr = lit(_parse_value(right))
            pipeline.derive(target, {
                "+": left_expr + right_expr,
                "-": left_expr - right_expr,
                "*": left_expr * right_expr,
                "/": left_expr / right_expr,
            }[op])
            return
    if formula.isidentifier():
        pipeline.derive(target, col(formula))
        return
    pipeline.derive(target, lit(_parse_value(formula)))


def _rename(pipeline, rest):
    mapping = {}
    for pair in rest.split(","):
        old, arrow, new = pair.partition("->")
        old = old.strip()
        new = new.strip()
        if not arrow or not old.isidentifier() or not new.isidentifier():
            raise PipelineError("rename needs 'old -> new' pairs")
        mapping[old] = new
    pipeline.rename(mapping)


def _select(pipeline, rest):
    columns = [c.strip() for c in rest.split(",")]
    for column in columns:
        if not column.isidentifier():
            raise PipelineError("bad column name %r" % column)
    pipeline.select(*columns)


def _sort(pipeline, rest):
    descending = False
    if rest.endswith(" desc"):
        descending = True
        rest = rest[: -len(" desc")]
    keys = [k.strip() for k in rest.split(",")]
    for key in keys:
        if not key.isidentifier():
            raise PipelineError("bad column name %r" % key)
    pipeline.sort(*keys, descending=descending)


def _limit(pipeline, rest):
    try:
        count = int(rest)
    except ValueError:
        raise PipelineError("limit needs an integer")
    pipeline.limit(count)


_HANDLERS = {
    "filter": _filter,
    "derive": _derive,
    "rename": _rename,
    "select": _select,
    "sort": _sort,
    "limit": _limit,
}

path: src/queries.py

content: """Rendering results for terminals and logs."""


def format_table(rows, columns=None):
    """Render ``rows`` as fixed-width text lines, header first.

    ``columns`` defaults to the first row's columns in order. Column
    widths fit the widest cell, so output is stable for stable input.
    """
    if not rows:
        return ["(no rows)"]
    if columns is None:
        columns = list(rows[0])
    widths = {c: len(c) for c in columns}
    rendered_rows = []
    for row in rows:
        rendered = {c: _cell(row.get(c)) for c in columns}
        for c in columns:
            widths[c] = max(widths[c], len(rendered[c]))
        rendered_rows.append(rendered)
    header = "  ".join(c.ljust(widths[c]) for c in columns).rstrip()
    lines = [header]
    for rendered in rendered_rows:
        lines.append(
            "  ".join(rendered[c].ljust(widths[c]) for c in columns).rstrip()
        )
    return lines


def format_plan(lines):
    """Indent plan lines under a heading for log output."""
    return ["plan:"] + ["  " + line for line in lines]


def _cell(value):
    if value is None:
        return ""
    return str(value)

path: src/report.py

content: """Pipeline stages."""

from src.stages.aggregate import Aggregate
from src.stages.base import Stage
from src.stages.derive import Derive
from src.stages.filter import Filter
from src.stages.limit import Limit
from src.stages.rename import Rename
from src.stages.select import Select
from src.stages.sort import Sort

__all__ = [
    "Aggregate",
    "Derive",
    "Filter",
    "Limit",
    "Rename",
    "Select",
    "Sort",
    "Stage",
]

path: src/stages/__init__.py

content: """Grouped aggregation."""

from src import metrics
from src.errors import DuplicateColumn, InvalidStage
from src.stages.base import Stage

_AGG_FUNCS = {
    "sum": sum,
    "count": len,
    "min": min,
    "max": max,
}


class Aggregate(Stage):
    """Group rows by ``keys`` and compute one row per group.

    ``aggs`` maps each output column name to an ``(function, column)``
    pair; ``count`` ignores its column and counts rows. Output rows carry
    the key columns plus the aggregate columns and are emitted in key
    order, so aggregation results are deterministic.
    """

    def __init__(self, keys, aggs):
        keys = tuple(keys)
        if not keys:
            raise InvalidStage("aggregate requires at least one key column")
        if not aggs:
            raise InvalidStage("aggregate requires at least one aggregation")
        for name, (fn, _column) in aggs.items():
            if fn not in _AGG_FUNCS:
                raise InvalidStage("unknown aggregate function: %s" % fn)
            if name in keys:
                raise DuplicateColumn(
                    "aggregate output %s collides with a key column" % name
                )
        self.keys = keys
        self.aggs = dict(aggs)

    def apply(self, rows):
        metrics.observe("aggregate", len(rows))
        groups = {}
        for row in rows:
            key = tuple(row[k] for k in self.keys)
            groups.setdefault(key, []).append(row)
        out = []
        for key in sorted(groups):
            members = groups[key]
            new_row = dict(zip(self.keys, key))
            for name in self.aggs:
                fn, column = self.aggs[name]
                if fn == "count":
                    new_row[name] = len(members)
                else:
                    new_row[name] = _AGG_FUNCS[fn](
                        member[column] for member in members
                    )
            out.append(new_row)
        return out

    def output_columns(self, columns):
        return frozenset(self.keys) | frozenset(self.aggs)

    def describe(self):
        rendered = ", ".join(
            "%s := %s(%s)" % (name, self.aggs[name][0], self.aggs[name][1])
            for name in sorted(self.aggs)
        )
        return "aggregate[%s | %s]" % (", ".join(self.keys), rendered)

path: src/stages/aggregate.py

content: """Stage interface.

A stage is one step of a pipeline. Stages are pure: :meth:`Stage.apply`
returns a new list of new row dicts and never mutates its input. That
purity is what lets the planner rearrange stages without worrying about
shared state.
"""


class Stage:
    """One transformation step in a pipeline."""

    def apply(self, rows):
        """Transform ``rows`` (a list of dicts) into a new list of dicts."""
        raise NotImplementedError

    def output_columns(self, columns):
        """Given the columns entering this stage, return the columns leaving it."""
        return frozenset(columns)

    def describe(self):
        """One stable line describing this stage, used by plan explains."""
        raise NotImplementedError

    def __repr__(self):
        return "<%s>" % self.describe()

path: src/stages/base.py

content: """Computed columns."""

from src import metrics
from src.errors import InvalidStage
from src.expr import Expr, evaluate, render
from src.stages.base import Stage


class Derive(Stage):
    """Compute ``expression`` for every row and store it in column ``target``.

    If a column called ``target`` already exists, its value is replaced;
    otherwise the column is added. The inputs to ``expression`` are always
    the values entering this stage, so a derive may redefine a column in
    terms of its own previous value (``score := score * 2``).
    """

    def __init__(self, target, expression):
        if not isinstance(target, str) or not target:
            raise InvalidStage("derive target must be a non-empty column name")
        if not isinstance(expression, Expr):
            raise InvalidStage("derive expression must be an expression")
        self.target = target
        self.expression = expression

    def apply(self, rows):
        metrics.observe("derive", len(rows))
        out = []
        for row in rows:
            new_row = dict(row)
            new_row[self.target] = evaluate(self.expression, row)
            out.append(new_row)
        return out

    def output_columns(self, columns):
        return frozenset(columns) | {self.target}

    def describe(self):
        return "derive[%s := %s]" % (self.target, render(self.expression))

path: src/stages/derive.py

content: """Row filtering."""

from src import metrics
from src.errors import InvalidStage
from src.expr import Expr, evaluate, render
from src.stages.base import Stage


class Filter(Stage):
    """Keep only the rows for which ``predicate`` evaluates truthy."""

    def __init__(self, predicate):
        if not isinstance(predicate, Expr):
            raise InvalidStage("filter predicate must be an expression")
        self.predicate = predicate

    def apply(self, rows):
        metrics.observe("filter", len(rows))
        return [dict(row) for row in rows if evaluate(self.predicate, row)]

    def describe(self):
        return "filter[%s]" % render(self.predicate)

path: src/stages/filter.py

content: """Row limiting."""

from src import metrics
from src.errors import InvalidStage
from src.stages.base import Stage


class Limit(Stage):
    """Keep only the first ``count`` rows.

    Unlike a filter, which rows survive depends on their position, so a
    limit pins everything before it in place: reordering work across a
    limit changes which rows it keeps.
    """

    def __init__(self, count):
        if not isinstance(count, int) or count < 0:
            raise InvalidStage("limit must be a non-negative integer")
        self.count = count

    def apply(self, rows):
        metrics.observe("limit", len(rows))
        return [dict(row) for row in rows[: self.count]]

    def describe(self):
        return "limit[%d]" % self.count

path: src/stages/limit.py

content: """Column renaming."""

from src import metrics
from src.errors import DuplicateColumn, InvalidStage
from src.stages.base import Stage


class Rename(Stage):
    """Rename columns according to ``mapping`` (old name -> new name).

    A rename is a pure relabeling: no values change and no rows change —
    only column names do. All entries in the mapping are applied
    simultaneously, so swaps and rotations are well defined
    (``{"a": "b", "b": "a"}`` exchanges two columns). Because a rename
    never touches data, work can be moved across one by translating
    column references through the mapping.
    """

    def __init__(self, mapping):
        if not isinstance(mapping, dict) or not mapping:
            raise InvalidStage("rename requires a non-empty mapping")
        new_names = list(mapping.values())
        if len(set(new_names)) != len(new_names):
            raise DuplicateColumn(
                "rename maps multiple columns to the same name"
            )
        self.mapping = dict(mapping)

    def apply(self, rows):
        metrics.observe("rename", len(rows))
        return [
            {self.mapping.get(name, name): value for name, value in row.items()}
            for row in rows
        ]

    def output_columns(self, columns):
        columns = frozenset(columns)
        renamed = {old for old in self.mapping if old in columns}
        return (columns - renamed) | {
            self.mapping[old] for old in renamed
        }

    def describe(self):
        pairs = ", ".join(
            "%s -> %s" % (old, self.mapping[old]) for old in sorted(self.mapping)
        )
        return "rename[%s]" % pairs

path: src/stages/rename.py

content: """Column projection."""

from src import metrics
from src.errors import InvalidStage
from src.stages.base import Stage


class Select(Stage):
    """Keep only the listed columns, in the listed order."""

    def __init__(self, columns):
        columns = tuple(columns)
        if not columns:
            raise InvalidStage("select requires at least one column")
        if len(set(columns)) != len(columns):
            raise InvalidStage("select lists a column twice")
        self.columns = columns

    def apply(self, rows):
        metrics.observe("select", len(rows))
        return [{name: row[name] for name in self.columns} for row in rows]

    def output_columns(self, columns):
        return frozenset(self.columns)

    def describe(self):
        return "select[%s]" % ", ".join(self.columns)

path: src/stages/select.py

content: """Row ordering."""

from src import metrics
from src.errors import InvalidStage
from src.stages.base import Stage


class Sort(Stage):
    """Sort rows by the given key columns.

    The sort is stable and row-preserving: every input row appears exactly
    once in the output, in key order, with ties left in their incoming
    order.
    """

    def __init__(self, keys, descending=False):
        keys = tuple(keys)
        if not keys:
            raise InvalidStage("sort requires at least one key column")
        self.keys = keys
        self.descending = bool(descending)

    def apply(self, rows):
        metrics.observe("sort", len(rows))
        return sorted(
            (dict(row) for row in rows),
            key=lambda row: tuple(row[k] for k in self.keys),
            reverse=self.descending,
        )

    def describe(self):
        rendered = ", ".join(self.keys)
        if self.descending:
            rendered += " desc"
        return "sort[%s]" % rendered

path: src/stages/sort.py

content: """Pipeline validation.

Validation runs against the pipeline as the user wrote it, before any
optimization, and checks that every stage only refers to columns that
actually exist at that point. Catching these mistakes up front means
the engine and the planner can assume every plan they see is
well-formed.
"""

from src.errors import DuplicateColumn, UnknownColumn
from src.expr import references
from src.stages.aggregate import Aggregate
from src.stages.derive import Derive
from src.stages.filter import Filter
from src.stages.rename import Rename
from src.stages.select import Select
from src.stages.sort import Sort


def check(stages, source_columns):
    """Raise a :class:`PipelineError` if ``stages`` is not well-formed."""
    available = frozenset(source_columns)
    for position, stage in enumerate(stages):
        _check_stage(stage, available, position)
        available = stage.output_columns(available)


def _missing(names, available):
    return sorted(frozenset(names) - available)


def _fail_unknown(stage, missing, position):
    raise UnknownColumn(
        "step %d (%s) refers to unknown column(s): %s"
        % (position + 1, stage.describe(), ", ".join(missing))
    )


def _check_stage(stage, available, position):
    if isinstance(stage, Filter):
        missing = _missing(references(stage.predicate), available)
        if missing:
            _fail_unknown(stage, missing, position)
    elif isinstance(stage, Derive):
        missing = _missing(references(stage.expression), available)
        if missing:
            _fail_unknown(stage, missing, position)
    elif isinstance(stage, Rename):
        missing = _missing(stage.mapping, available)
        if missing:
            _fail_unknown(stage, missing, position)
        survivors = available - frozenset(stage.mapping)
        collisions = sorted(
            new for new in stage.mapping.values() if new in survivors
        )
        if collisions:
            raise DuplicateColumn(
                "step %d (%s) would produce duplicate column(s): %s"
                % (position + 1, stage.describe(), ", ".join(collisions))
            )
    elif isinstance(stage, Select):
        missing = _missing(stage.columns, available)
        if missing:
            _fail_unknown(stage, missing, position)
    elif isinstance(stage, Sort):
        missing = _missing(stage.keys, available)
        if missing:
            _fail_unknown(stage, missing, position)
    elif isinstance(stage, Aggregate):
        needed = set(stage.keys)
        for name in stage.aggs:
            fn, column = stage.aggs[name]
            if fn != "count":
                needed.add(column)
        missing = _missing(needed, available)
        if missing:
            _fail_unknown(stage, missing, position)

path: src/validate.py

content: """Tests for the expression module."""

import pytest

from src.errors import UnknownColumn
from src.expr import col, evaluate, func, lit, references, render


def test_arithmetic_and_comparison():
    row = {"price": 10, "quantity": 3}
    total = col("price") * col("quantity")
    assert evaluate(total, row) == 30
    assert evaluate(total >= 30, row) is True
    assert evaluate((col("price") + 5) / 3, row) == 5
    assert evaluate(col("price") != 10, row) is False


def test_boolean_composition_and_wrapping():
    row = {"status": "active", "score": 7}
    keep = (col("status") == "active") & (col("score") > 10)
    assert evaluate(keep, row) is False
    either = (col("status") == "active") | (col("score") > 10)
    assert evaluate(either, row) is True
    assert evaluate(~(col("score") > 10), row) is True


def test_functions():
    row = {"name": "Anna", "delta": -4}
    assert evaluate(func("upper", col("name")), row) == "ANNA"
    assert evaluate(func("length", col("name")), row) == 4
    assert evaluate(func("abs", col("delta")), row) == 4


def test_references_walks_nested_trees():
    predicate = ((col("a") + col("b")) > 3) & (~(col("c") == "x"))
    assert references(predicate) == frozenset({"a", "b", "c"})
    assert references(lit(5)) == frozenset()


def test_render_is_stable():
    assert render(col("score") >= 50) == "(score >= 50)"
    assert (
        render((col("status") == "active") & (col("n") > 3))
        == "((status == 'active') and (n > 3))"
    )
    assert render(func("upper", col("name"))) == "upper(name)"
    assert render(col("price") * 2) == "(price * 2)"


def test_missing_column_raises():
    with pytest.raises(UnknownColumn):
        evaluate(col("absent") > 1, {"present": 1})

path: tests/public/test_expr.py

content: """End-to-end pipeline tests."""

import pytest

from src import metrics
from src.errors import DuplicateColumn, PipelineError, UnknownColumn
from src.expr import col
from src.pipeline import Pipeline

ORDERS = [
    {"id": 1, "status": "active", "price": 10, "quantity": 2},
    {"id": 2, "status": "closed", "price": 99, "quantity": 9},
    {"id": 3, "status": "active", "price": 5, "quantity": 3},
    {"id": 4, "status": "active", "price": 7, "quantity": 1},
]


def _totals_pipeline():
    return (
        Pipeline()
        .filter(col("status") == "active")
        .derive("total", col("price") * col("quantity"))
        .sort("total", descending=True)
        .limit(2)
    )


def test_run_optimized_matches_unoptimized():
    expected = [
        {"id": 1, "status": "active", "price": 10, "quantity": 2, "total": 20},
        {"id": 3, "status": "active", "price": 5, "quantity": 3, "total": 15},
    ]
    assert _totals_pipeline().run(ORDERS) == expected
    assert _totals_pipeline().run(ORDERS, optimize=False) == expected


def test_rename_then_filter_matches_unoptimized():
    p = (
        Pipeline()
        .rename({"quantity": "units"})
        .filter(col("status") == "active")
        .select("id", "units")
    )
    expected = [{"id": 1, "units": 2}, {"id": 3, "units": 3}, {"id": 4, "units": 1}]
    assert p.run(ORDERS) == expected
    q = (
        Pipeline()
        .rename({"quantity": "units"})
        .filter(col("status") == "active")
        .select("id", "units")
    )
    assert q.run(ORDERS, optimize=False) == expected


def test_explain_lists_each_step():
    lines = _totals_pipeline().explain(["id", "status", "price", "quantity"])
    assert lines == [
        "scan[id, status, price, quantity]",
        "filter[(status == 'active')]",
        "derive[total := (price * quantity)]",
        "sort[total desc]",
        "limit[2]",
    ]


def test_adjacent_filters_fuse():
    p = (
        Pipeline()
        .filter(col("status") == "active")
        .filter(col("price") > 6)
        .derive("total", col("price") * col("quantity"))
    )
    lines = p.explain(["id", "status", "price", "quantity"])
    assert lines == [
        "scan[id, status, price, quantity]",
        "filter[((status == 'active') and (price > 6))]",
        "derive[total := (price * quantity)]",
    ]
    assert [r["id"] for r in p.run(ORDERS)] == [1, 4]


def test_validation_errors():
    with pytest.raises(UnknownColumn) as exc:
        Pipeline().filter(col("missing") > 1).run(ORDERS)
    assert "unknown column" in str(exc.value)
    with pytest.raises(DuplicateColumn):
        Pipeline().rename({"price": "quantity"}).run(ORDERS)
    with pytest.raises(UnknownColumn):
        Pipeline().select("id", "absent").explain(["id", "status"])
    with pytest.raises(PipelineError):
        Pipeline().filter(col("id") > 0).run([])


def test_metrics_observe_rows_entering_each_stage():
    metrics.reset()
    Pipeline().filter(col("status") == "active").sort("id").run(
        ORDERS, optimize=False
    )
    assert metrics.counters["filter"] == 4
    assert metrics.counters["sort"] == 3
    metrics.reset()
    assert metrics.snapshot() == {}

path: tests/public/test_pipeline.py

content: """Tests for individual stages."""

import pytest

from src.errors import DuplicateColumn
from src.expr import col
from src.stages import (
    Aggregate,
    Derive,
    Filter,
    Limit,
    Rename,
    Select,
    Sort,
)


def test_filter_keeps_matches_and_copies():
    rows = [{"id": 1, "score": 5}, {"id": 2, "score": 9}]
    out = Filter(col("score") > 6).apply(rows)
    assert out == [{"id": 2, "score": 9}]
    out[0]["score"] = 0
    assert rows[1]["score"] == 9


def test_derive_adds_and_replaces():
    rows = [{"id": 1, "score": 10}]
    added = Derive("double", col("score") * 2).apply(rows)
    assert added == [{"id": 1, "score": 10, "double": 20}]
    replaced = Derive("score", col("score") * 2).apply(rows)
    assert replaced == [{"id": 1, "score": 20}]
    assert rows == [{"id": 1, "score": 10}]


def test_rename_simple_and_rotation():
    rows = [{"cost": 3, "price": 9}]
    assert Rename({"cost": "base"}).apply(rows) == [{"base": 3, "price": 9}]
    rotated = Rename({"cost": "price", "price": "msrp"}).apply(rows)
    assert rotated == [{"price": 3, "msrp": 9}]
    swap = Rename({"cost": "price", "price": "cost"}).apply(rows)
    assert swap == [{"price": 3, "cost": 9}]


def test_select_and_limit():
    rows = [{"a": 1, "b": 2, "c": 3}, {"a": 4, "b": 5, "c": 6}]
    assert Select(("c", "a")).apply(rows) == [{"c": 3, "a": 1}, {"c": 6, "a": 4}]
    assert Limit(1).apply(rows) == [{"a": 1, "b": 2, "c": 3}]


def test_sort_is_stable():
    rows = [
        {"id": "x", "rank": 2},
        {"id": "y", "rank": 1},
        {"id": "z", "rank": 2},
    ]
    ordered = Sort(("rank",)).apply(rows)
    assert [r["id"] for r in ordered] == ["y", "x", "z"]
    reverse = Sort(("rank",), descending=True).apply(rows)
    assert [r["id"] for r in reverse] == ["x", "z", "y"]


def test_aggregate_groups_deterministically():
    rows = [
        {"region": "us", "amount": 10},
        {"region": "eu", "amount": 5},
        {"region": "us", "amount": 7},
    ]
    stage = Aggregate(
        ("region",), {"total": ("sum", "amount"), "n": ("count", "region")}
    )
    assert stage.apply(rows) == [
        {"region": "eu", "total": 5, "n": 1},
        {"region": "us", "total": 17, "n": 2},
    ]
    with pytest.raises(DuplicateColumn):
        Aggregate(("region",), {"region": ("sum", "amount")})


def test_describe_strings():
    assert Filter(col("score") >= 50).describe() == "filter[(score >= 50)]"
    assert (
        Derive("score", col("score") * 2).describe()
        == "derive[score := (score * 2)]"
    )
    assert (
        Rename({"cost": "price", "price": "msrp"}).describe()
        == "rename[cost -> price, price -> msrp]"
    )
    assert Select(("id", "name")).describe() == "select[id, name]"
    assert Sort(("score",), descending=True).describe() == "sort[score desc]"
    assert Limit(10).describe() == "limit[10]"
    assert (
        Aggregate(("region",), {"total": ("sum", "amount")}).describe()
        == "aggregate[region | total := sum(amount)]"
    )

path: tests/public/test_stages.py

content: """Tests for the datasource, report, queries, and costs modules."""

import pytest

from src import costs
from src.datasource import rows_from_text
from src.errors import PipelineError
from src.expr import col
from src.queries import compile_text
from src.report import format_plan, format_table
from src.stages import Derive, Filter, Sort

EXPORT = """
id | name | score
1  | anna | 10
2  | ben  | 7
"""


def test_rows_from_text_with_types():
    rows = rows_from_text(EXPORT, types={"id": "int", "score": "int"})
    assert rows == [
        {"id": 1, "name": "anna", "score": 10},
        {"id": 2, "name": "ben", "score": 7},
    ]


def test_rows_from_text_rejects_ragged_lines():
    with pytest.raises(PipelineError) as exc:
        rows_from_text("a | b\n1 | 2 | 3")
    assert "expected" in str(exc.value)
    with pytest.raises(PipelineError):
        rows_from_text("a | b\nx | 1", types={"b": "bool"})


def test_format_table_and_plan():
    rows = [{"id": 1, "name": "anna"}, {"id": 2, "name": "b"}]
    assert format_table(rows) == [
        "id  name",
        "1   anna",
        "2   b",
    ]
    assert format_table([]) == ["(no rows)"]
    assert format_plan(["scan[a]", "filter[(a > 1)]"]) == [
        "plan:",
        "  scan[a]",
        "  filter[(a > 1)]",
    ]


def test_compile_text_builds_the_written_pipeline():
    text = """
    filter status == 'active' and score >= 5
    derive total = score * 3
    sort total desc
    limit 2
    """
    pipeline = compile_text(text)
    assert [stage.describe() for stage in pipeline.stages()] == [
        "filter[((status == 'active') and (score >= 5))]",
        "derive[total := (score * 3)]",
        "sort[total desc]",
        "limit[2]",
    ]
    rows = [
        {"status": "active", "score": 6},
        {"status": "idle", "score": 9},
        {"status": "active", "score": 4},
        {"status": "active", "score": 8},
    ]
    assert [r["total"] for r in pipeline.run(rows)] == [24, 18]


def test_compile_text_rejects_unknown_steps():
    with pytest.raises(PipelineError) as exc:
        compile_text("explode everything")
    assert "line 1" in str(exc.value)
    with pytest.raises(PipelineError):
        compile_text("limit soon")


def test_estimates_reward_early_pruning():
    filter_first = [Filter(col("flag") == 1), Derive("x", col("a") + 1)]
    derive_first = [Derive("x", col("a") + 1), Filter(col("flag") == 1)]
    assert costs.estimate(filter_first, 100) == 300
    assert costs.estimate(derive_first, 100) == 500
    assert costs.breakdown(filter_first, 100) == [
        ("filter[(flag == 1)]", 100, 100),
        ("derive[x := (a + 1)]", 50, 200),
    ]


def test_sort_weight_and_limit_rows():
    stages = [Sort(("a",)), Filter(col("a") > 0)]
    assert costs.estimate(stages, 10) == 30
    assert costs.rows_leaving(Filter(col("a") > 0), 9) == 4

path: tests/public/test_tools.py

Reference patch:
new file mode 100644
--- /dev/null
+++ b/src/planner/analysis.py
@@ -0,0 +1,50 @@
+"""Field-level analysis for plan rewrites.
+
+A filter may only move to the other side of a stage if every column it
+reads still means the same thing there — the name being *present* is
+not enough, because a stage can rebind a name to a different value. A
+derive rebinds its target column. A rename rebinds names on both sides
+of its mapping while preserving values, so a reference can be
+translated through the mapping instead of blocking the move.
+"""
+
+from src.expr import And, Arith, Cmp, Col, Func, Lit, Not, Or
+
+
+def translate(expression, mapping):
+    """Rewrite every column reference in ``expression`` per ``mapping``.
+
+    Names missing from ``mapping`` are left as they are. The input
+    expression is not modified.
+    """
+    if isinstance(expression, Col):
+        return Col(mapping.get(expression.name, expression.name))
+    if isinstance(expression, Lit):
+        return expression
+    if isinstance(expression, Arith):
+        return Arith(
+            expression.op,
+            translate(expression.left, mapping),
+            translate(expression.right, mapping),
+        )
+    if isinstance(expression, Cmp):
+        return Cmp(
+            expression.op,
+            translate(expression.left, mapping),
+            translate(expression.right, mapping),
+        )
+    if isinstance(expression, And):
+        return And(
+            translate(expression.left, mapping),
+            translate(expression.right, mapping),
+        )
+    if isinstance(expression, Or):
+        return Or(
+            translate(expression.left, mapping),
+            translate(expression.right, mapping),
+        )
+    if isinstance(expression, Not):
+        return Not(translate(expression.operand, mapping))
+    if isinstance(expression, Func):
+        return Func(expression.name, translate(expression.operand, mapping))
+    return expression
--- a/src/planner/optimizer.py
+++ b/src/planner/optimizer.py
@@ -50,8 +50,9 @@ def _push_filters(stages, source_columns):
             prev = stages[i - 1]
             if not isinstance(filt, Filter) or isinstance(prev, Filter):
                 continue
-            if rules.can_push_filter_below(filt, prev, entering[i - 1]):
-                stages[i - 1], stages[i] = filt, prev
+            pushed = rules.push_filter_below(filt, prev, entering[i - 1])
+            if pushed is not None:
+                stages[i - 1], stages[i] = pushed, prev
                 moved = True
                 break
     return stages
--- a/src/planner/rules.py
+++ b/src/planner/rules.py
@@ -7,25 +7,40 @@ cheaper and returns the same rows.
 """
 
 from src.expr import references
+from src.planner.analysis import translate
 from src.stages.derive import Derive
+from src.stages.filter import Filter
 from src.stages.rename import Rename
 from src.stages.select import Select
 from src.stages.sort import Sort
 
-#: Stages that neither add nor remove rows. Filtering before one of
-#: these keeps exactly the same set of rows in play, so each row's
-#: filter verdict is unaffected by which side of the stage it runs on.
+#: Stages that neither add nor remove rows. Only these are candidates
+#: for letting a filter cross; whether a particular filter may do so
+#: additionally depends on which columns the stage rebinds.
 ROW_PRESERVING = (Derive, Rename, Select, Sort)
 
 
-def can_push_filter_below(filt, stage, upstream_columns):
-    """Return True when ``filt`` may run before ``stage``.
+def push_filter_below(filt, stage, upstream_columns):
+    """Return the filter to run before ``stage``, or ``None``.
 
-    A filter can only be evaluated at a point in the plan where every
-    column it mentions is present. The move is therefore allowed when
-    the preceding stage is row-preserving and all of the filter's
-    columns are already available upstream of it.
+    A move is safe only when every column the filter reads means the
+    same thing on the other side of the stage — presence of the name is
+    not enough. A derive rebinds its target column, so a filter that
+    reads the target must not cross it. A rename rebinds names while
+    preserving the underlying values, so the filter's references are
+    translated through the mapping and the translated filter crosses
+    instead.
     """
     if not isinstance(stage, ROW_PRESERVING):
-        return False
-    return references(filt.predicate) <= frozenset(upstream_columns)
+        return None
+    predicate = filt.predicate
+    if isinstance(stage, Derive) and stage.target in references(predicate):
+        return None
+    if isinstance(stage, Rename):
+        inverse = {new: old for old, new in stage.mapping.items()}
+        predicate = translate(predicate, inverse)
+    if not references(predicate) <= frozenset(upstream_columns):
+        return None
+    if predicate is filt.predicate:
+        return filt
+    return Filter(predicate)


Hidden acceptance tests (diff):
diff --git a/tests/hidden/test_hidden.py b/tests/hidden/test_hidden.py
new file mode 100644
--- /dev/null
+++ b/tests/hidden/test_hidden.py
@@ -0,0 +1,267 @@
+"""Hidden evaluation tests. Never agent-visible (DEC-015).
+
+FAIL_TO_PASS: the planner decides filter pushdown by column *presence*
+(is the name defined upstream?) when the safe condition is column
+*meaning* (does the name still hold the same value?). Filters pushed
+across a derive that overwrites the column they read, or across a
+rename that rebinds a name, evaluate against the wrong values — and
+filters that could legally cross a rename (with their references
+translated through the mapping) are never pushed at all. Every test
+below fails at baseline: the correctness tests through wrong rows, the
+plan-shape/work tests through the missing rename translation or the
+missed pushdown.
+
+Graded gradient (DEC-036/DEC-042): disabling pushdown outright passes
+only the three pure row-correctness tests (3/10); dropping Derive and
+Rename from the type list also recovers the derive-then-sort placement
+(4/10); a field-aware fix that blocks renames instead of translating
+references lands 7/10. Only a fix that analyses reads/writes per stage
+AND translates predicates across renames passes 10/10.
+"""
+
+from src import metrics
+from src.expr import col
+from src.pipeline import Pipeline
+
+
+def test_filter_after_overwriting_derive_uses_final_values():
+    rows = [
+        {"id": 1, "score": 30},
+        {"id": 2, "score": 45},
+        {"id": 3, "score": 50},
+        {"id": 4, "score": 80},
+        {"id": 5, "score": 100},
+        {"id": 6, "score": 120},
+    ]
+    p = Pipeline().derive("score", col("score") * 2).filter(col("score") >= 100)
+    out = p.run(rows)
+    assert [r["id"] for r in out] == [3, 4, 5, 6]
+    assert [r["score"] for r in out] == [100, 160, 200, 240]
+
+
+def test_filter_after_rotating_rename_reads_rebound_column():
+    rows = [
+        {"sku": "A", "cost": 25, "price": 60},
+        {"sku": "B", "cost": 55, "price": 30},
+        {"sku": "C", "cost": 10, "price": 90},
+    ]
+    p = (
+        Pipeline()
+        .rename({"cost": "price", "price": "msrp"})
+        .filter(col("price") < 40)
+        .select("sku")
+    )
+    assert p.run(rows) == [{"sku": "A"}, {"sku": "C"}]
+
+
+def test_filter_crosses_sort_but_stops_at_overwriting_derive():
+    rows = [
+        {"id": 1, "bonus": 10},
+        {"id": 2, "bonus": 25},
+        {"id": 3, "bonus": 30},
+        {"id": 4, "bonus": 5},
+    ]
+    p = (
+        Pipeline()
+        .derive("bonus", col("bonus") * 2)
+        .sort("bonus")
+        .filter(col("bonus") >= 40)
+    )
+    assert p.run(rows) == [{"id": 2, "bonus": 50}, {"id": 3, "bonus": 60}]
+    assert p.explain(["id", "bonus"]) == [
+        "scan[id, bonus]",
+        "derive[bonus := (bonus * 2)]",
+        "filter[(bonus >= 40)]",
+        "sort[bonus]",
+    ]
+
+
+def test_compound_predicate_sees_final_values():
+    rows = [
+        {"id": 1, "tier": "pro", "score": 50, "boost": 20},
+        {"id": 2, "tier": "pro", "score": 55, "boost": 0},
+        {"id": 3, "tier": "free", "score": 90, "boost": 10},
+        {"id": 4, "tier": "pro", "score": 40, "boost": 25},
+        {"id": 5, "tier": "pro", "score": 65, "boost": -30},
+    ]
+    p = (
+        Pipeline()
+        .derive("score", col("score") + col("boost"))
+        .filter((col("tier") == "pro") & (col("score") >= 60))
+        .select("id")
+    )
+    assert p.run(rows) == [{"id": 1}, {"id": 4}]
+
+
+def test_safe_pushdown_survives_next_to_unsafe_one():
+    rows = [
+        {"id": 1, "region": "eu", "score": 60, "penalty": 55},
+        {"id": 2, "region": "eu", "score": 40, "penalty": 10},
+        {"id": 3, "region": "us", "score": 80, "penalty": 0},
+        {"id": 4, "region": "eu", "score": 55, "penalty": 40},
+    ]
+    p = (
+        Pipeline()
+        .derive("flag", col("score") >= 50)
+        .filter(col("region") == "eu")
+        .derive("score", col("score") - col("penalty"))
+        .filter(col("score") >= 10)
+        .select("id", "flag", "score")
+    )
+    assert p.run(rows) == [
+        {"id": 2, "flag": False, "score": 30},
+        {"id": 4, "flag": True, "score": 15},
+    ]
+    assert p.explain(["id", "region", "score", "penalty"]) == [
+        "scan[id, region, score, penalty]",
+        "filter[(region == 'eu')]",
+        "derive[flag := (score >= 50)]",
+        "derive[score := (score - penalty)]",
+        "filter[(score >= 10)]",
+        "select[id, flag, score]",
+    ]
+
+
+def test_filter_on_renamed_column_is_pushed_with_translation():
+    rows = [
+        {"txn": "t1", "amt": 250},
+        {"txn": "t2", "amt": 40},
+        {"txn": "t3", "amt": 180},
+    ]
+    p = (
+        Pipeline()
+        .rename({"amt": "amount"})
+        .filter(col("amount") > 100)
+        .select("txn", "amount")
+    )
+    assert p.run(rows) == [
+        {"txn": "t1", "amount": 250},
+        {"txn": "t3", "amount": 180},
+    ]
+    assert p.explain(["txn", "amt"]) == [
+        "scan[txn, amt]",
+        "filter[(amt > 100)]",
+        "rename[amt -> amount]",
+        "select[txn, amount]",
+    ]
+
+
+def test_translation_carries_through_rename_and_sort():
+    rows = [
+        {"ev": "boot", "ts": 500},
+        {"ev": "login", "ts": 120},
+        {"ev": "sync", "ts": 300},
+    ]
+    p = (
+        Pipeline()
+        .rename({"ts": "stamp"})
+        .sort("stamp")
+        .filter(col("stamp") <= 300)
+    )
+    assert p.run(rows) == [
+        {"ev": "login", "stamp": 120},
+        {"ev": "sync", "stamp": 300},
+    ]
+    assert p.explain(["ev", "ts"]) == [
+        "scan[ev, ts]",
+        "filter[(ts <= 300)]",
+        "rename[ts -> stamp]",
+        "sort[stamp]",
+    ]
+
+
+def test_moved_filter_fuses_with_stationary_one():
+    rows = [
+        {"id": 1, "region": "eu", "tier": "gold", "score": 120, "weight": 1},
+        {"id": 2, "region": "eu", "tier": "gold", "score": 60, "weight": 3},
+        {"id": 3, "region": "eu", "tier": "gold", "score": 150, "weight": 0},
+        {"id": 4, "region": "eu", "tier": "silver", "score": 200, "weight": 2},
+        {"id": 5, "region": "us", "tier": "gold", "score": 100, "weight": 2},
+    ]
+    p = (
+        Pipeline()
+        .filter(col("region") == "eu")
+        .derive("flag", col("score") >= 90)
+        .filter(col("tier") == "gold")
+        .derive("score", col("score") * col("weight"))
+        .filter(col("score") >= 100)
+        .select("id", "score")
+    )
+    assert p.run(rows) == [{"id": 1, "score": 120}, {"id": 2, "score": 180}]
+    assert p.explain(["id", "region", "tier", "score", "weight"]) == [
+        "scan[id, region, tier, score, weight]",
+        "filter[((region == 'eu') and (tier == 'gold'))]",
+        "derive[flag := (score >= 90)]",
+        "derive[score := (score * weight)]",
+        "filter[(score >= 100)]",
+        "select[id, score]",
+    ]
+
+
+def test_full_plan_places_every_filter_correctly():
+    rows = [
+        {"id": 1, "amt": 50, "cat": "retail", "rate": 3},
+        {"id": 2, "amt": 200, "cat": "retail", "rate": 0.4},
+        {"id": 3, "amt": 120, "cat": "retail", "rate": 1},
+        {"id": 4, "amt": 300, "cat": "wholesale", "rate": 2},
+        {"id": 5, "amt": 80, "cat": "retail", "rate": 2},
+        {"id": 6, "amt": 40, "cat": "retail", "rate": 5},
+    ]
+    p = (
+        Pipeline()
+        .rename({"amt": "amount", "cat": "category"})
+        .derive("amount", col("amount") * col("rate"))
+        .filter(col("category") == "retail")
+        .filter(col("amount") >= 100)
+        .sort("amount", descending=True)
+        .limit(3)
+        .select("id", "amount", "category")
+    )
+    assert p.run(rows) == [
+        {"id": 6, "amount": 200, "category": "retail"},
+        {"id": 5, "amount": 160, "category": "retail"},
+        {"id": 1, "amount": 150, "category": "retail"},
+    ]
+    assert p.explain(["id", "amt", "cat", "rate"]) == [
+        "scan[id, amt, cat, rate]",
+        "filter[(cat == 'retail')]",
+        "rename[amt -> amount, cat -> category]",
+        "derive[amount := (amount * rate)]",
+        "filter[(amount >= 100)]",
+        "sort[amount desc]",
+        "limit[3]",
+        "select[id, amount, category]",
+    ]
+
+
+def test_optimized_run_actually_prunes_before_deriving():
+    scores = [40, 60, 110, 30, 55, 20, 90, 120]
+    rows = []
+    for i in range(1, 41):
+        rows.append(
+            {
+                "id": i,
+                "region": "eu" if i <= 8 else "us",
+                "amount": 10 * i,
+                "rate": 2,
+                "score": scores[i - 1] if i <= 8 else 10,
+            }
+        )
+    p = (
+        Pipeline()
+        .derive("total", col("amount") * col("rate"))
+        .filter(col("region") == "eu")
+        .derive("score", col("score") * 2)
+        .filter(col("score") >= 100)
+        .select("id", "score")
+    )
+    metrics.reset()
+    out = p.run(rows)
+    assert out == [
+        {"id": 2, "score": 120},
+        {"id": 3, "score": 220},
+        {"id": 5, "score": 110},
+        {"id": 7, "score": 180},
+        {"id": 8, "score": 240},
+    ]
+    assert metrics.counters["derive"] <= 16