Repository: ibis-project/ibis

Issue:
Scalar expressions involving filtered tables can yield incorrect SQL queries
``` python
In [12]: table = ibis.table([('flag', 'string'), ('unique_user', 'int64')], 'tbl')

In [13]: flagged = table[table.flag == '1'].unique_user.approx_nunique()

In [14]: total = table[table.flag == '0'].unique_user.approx_nunique()

In [15]: print (flagged / total).compile()
SELECT ndv(`unique_user`) / ndv(`unique_user`) AS `tmp`
FROM tbl
WHERE `flag` = '1'
```

There is an implicit join between inline views; will be tricky to make work but an important use case. 



Discussion:
I'm punting this to a 0.6.x bugfix release since I want to investigate the general issue (extracting scalar aggregates to inline views) more thoroughly. 


Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -104,6 +104,113 @@ def _sub(self, expr):
         return helper.get_result()
 
 
+class ScalarAggregate(object):
+
+    def __init__(self, expr, memo=None, default_name='tmp'):
+        self.expr = expr
+        self.memo = memo or {}
+        self.tables = []
+        self.default_name = default_name
+
+    def get_result(self):
+        expr = self.expr
+
+        subbed_expr = self._visit(expr)
+
+        try:
+            name = subbed_expr.get_name()
+            named_expr = subbed_expr
+        except:
+            name = self.default_name
+            named_expr = subbed_expr.name(self.default_name)
+
+        tables = list(self.memo.values())
+        table = self.tables[0]
+        for other in self.tables[1:]:
+            table = table.cross_join(other)
+
+        return table.projection([named_expr]), name
+
+    def _visit(self, expr):
+        if is_scalar_reduce(expr) and not has_multiple_bases(expr):
+            # An aggregation unit
+            key = self._key(expr)
+            if key not in self.memo:
+                agg_expr, name = reduction_to_aggregation(expr)
+                self.memo[key] = agg_expr, name
+                self.tables.append(agg_expr)
+            else:
+                agg_expr, name = self.memo[key]
+            return agg_expr[name]
+
+        elif not isinstance(expr, ir.Expr):
+            return expr
+
+        node = expr.op()
+        subbed_args = []
+        for arg in node.args:
+            if isinstance(arg, (tuple, list)):
+                subbed_arg = [self._visit(x) for x in arg]
+            else:
+                subbed_arg = self._visit(arg)
+            subbed_args.append(subbed_arg)
+
+        subbed_node = type(node)(*subbed_args)
+        if isinstance(expr, ir.ValueExpr):
+            result = expr._factory(subbed_node, name=expr._name)
+        else:
+            result = expr._factory(subbed_node)
+
+        return result
+
+    def _key(self, expr):
+        return repr(expr.op())
+
+
+def has_multiple_bases(expr):
+    return len(find_all_tables(expr)) > 1
+
+
+def reduction_to_aggregation(expr, default_name='tmp'):
+    tables = find_all_tables(expr)
+
+    try:
+        name = expr.get_name()
+        named_expr = expr
+    except:
+        name = default_name
+        named_expr = expr.name(default_name)
+
+    if len(tables) == 1:
+        table = list(tables.values())[0]
+        return table.aggregate([named_expr]), name
+    else:
+        return ScalarAggregate(expr, None, default_name).get_result()
+
+
+def find_all_tables(expr, memo=None):
+    if memo is None:
+        memo = {}
+
+    node = expr.op()
+
+    if isinstance(expr, ir.TableExpr):
+        key = id(node)
+        if key not in memo:
+            memo[key] = expr
+        return memo
+
+    for arg in node.flat_args():
+        if isinstance(arg, ir.Expr):
+            find_all_tables(arg, memo)
+
+    return memo
+
+
+def is_scalar_reduce(x):
+    return isinstance(x, ir.ScalarExpr) and ops.is_reduction(x)
+
+
 def substitute_parents(expr, lift_memo=None, past_projection=True):
     rewriter = ExprSimplifier(expr, lift_memo=lift_memo,
                               block_projection=not past_projection)
diff --git a/ibis/expr/operations.py b/ibis/expr/operations.py
--- a/ibis/expr/operations.py
+++ b/ibis/expr/operations.py
@@ -60,7 +60,7 @@ def find_all_base_tables(expr, memo=None):
 
     if isinstance(expr, TableExpr) and node.blocks():
         if id(expr) not in memo:
-            memo[id(expr)] = expr
+            memo[id(node)] = expr
         return memo
 
     for arg in expr.op().flat_args():
@@ -1786,7 +1786,7 @@ def get_result(self):
             return self._attempt_pushdown()
 
     def _plain_subquery(self):
-        return Aggregation(self.selection, self.metrics,
+        return Aggregation(self.parent, self.metrics,
                            by=self.by, having=self.having)
 
     def _attempt_pushdown(self):
diff --git a/ibis/sql/compiler.py b/ibis/sql/compiler.py
--- a/ibis/sql/compiler.py
+++ b/ibis/sql/compiler.py
@@ -291,7 +291,7 @@ def _rewrite_reduction_filter(self, expr):
         # TODO: what about reductions that reference a join that isn't visible
         # at this level? Means we probably have the wrong design, but will have
         # to revisit when it becomes a problem.
-        aggregation, _ = _reduction_to_aggregation(expr, default_name='tmp')
+        aggregation, _ = L.reduction_to_aggregation(expr, default_name='tmp')
         return aggregation.to_array()
 
     def _visit_filter_Any(self, expr):
@@ -791,9 +791,6 @@ def as_is(x):
     if isinstance(expr, ir.TableExpr):
         return expr, as_is
 
-    def _scalar_reduce(x):
-        return isinstance(x, ir.ScalarExpr) and ops.is_reduction(x)
-
     def _get_scalar(field):
         def scalar_handler(results):
             return results[field][0]
@@ -801,8 +798,8 @@ def scalar_handler(results):
 
     if isinstance(expr, ir.ScalarExpr):
 
-        if _scalar_reduce(expr):
-            table_expr, name = _reduction_to_aggregation(
+        if L.is_scalar_reduce(expr):
+            table_expr, name = L.reduction_to_aggregation(
                 expr, default_name='tmp')
             return table_expr, _get_scalar(name)
         else:
@@ -823,7 +820,7 @@ def scalar_handler(results):
         any_aggregation = False
 
         for x in exprs:
-            if not _scalar_reduce(x):
+            if not L.is_scalar_reduce(x):
                 is_aggregation = False
             else:
                 any_aggregation = True
@@ -871,19 +868,6 @@ def column_handler(results):
                                    .format(type(expr)))
 
 
-def _reduction_to_aggregation(expr, default_name='tmp'):
-    table = ir.find_base_table(expr)
-
-    try:
-        name = expr.get_name()
-        named_expr = expr
-    except:
-        name = default_name
-        named_expr = expr.name(default_name)
-
-    return table.aggregate([named_expr]), name
-
-
 class QueryBuilder(object):
 
     select_builder = SelectBuilder


Test patch:
diff --git a/ibis/sql/tests/test_compiler.py b/ibis/sql/tests/test_compiler.py
--- a/ibis/sql/tests/test_compiler.py
+++ b/ibis/sql/tests/test_compiler.py
@@ -135,6 +135,51 @@ def test_simple_scalar_aggregates(self):
         output = DataFrame({'sum': [5]})
         assert handler(output) == 5
 
+    def test_scalar_aggregates_multiple_tables(self):
+        # #740
+        table = ibis.table([('flag', 'string'),
+                            ('value', 'double')],
+                           'tbl')
+
+        flagged = table[table.flag == '1']
+        unflagged = table[table.flag == '0']
+
+        expr = flagged.value.mean() / unflagged.value.mean() - 1
+
+        result = to_sql(expr)
+        expected = """\
+SELECT (t0.`mean` / t1.`mean`) - 1 AS `tmp`
+FROM (
+  SELECT avg(`value`) AS `mean`
+  FROM tbl
+  WHERE `flag` = '1'
+) t0
+  CROSS JOIN (
+    SELECT avg(`value`) AS `mean`
+    FROM tbl
+    WHERE `flag` = '0'
+  ) t1"""
+        assert result == expected
+
+        fv = flagged.value
+        uv = unflagged.value
+
+        expr = (fv.mean() / fv.sum()) - (uv.mean() / uv.sum())
+        result = to_sql(expr)
+        expected = """\
+SELECT t0.`tmp` - t1.`tmp` AS `tmp`
+FROM (
+  SELECT avg(`value`) / sum(`value`) AS `tmp`
+  FROM tbl
+  WHERE `flag` = '1'
+) t0
+  CROSS JOIN (
+    SELECT avg(`value`) / sum(`value`) AS `tmp`
+    FROM tbl
+    WHERE `flag` = '0'
+  ) t1"""
+        assert result == expected
+
     def test_table_column_unbox(self):
         from pandas import DataFrame