Repository: ibis-project/ibis

Issue:
BUG: Chained mutate operations give wrong results
Mutate function is giving wrong output. Please check below sample example. 

```import ibis 
table_expr = ibis.table([('a', 'string'), ('b', 'string')]) 
table_expr1 = table_expr['b'].fillna('Short Term') 
table_expr = table_expr.mutate(b = table_expr1) 
table_expr2 = table_expr['a'].fillna('Short Term') 
table_expr = table_expr.mutate(a = table_expr2) 
print(ibis.impala.compile(table_expr)) 
```
For the above program output is :
`SELECT isnull(`a`, 'Short Term') AS `a`, `b`
FROM t0`

instead of 
`SELECT isnull(`a`, 'Short Term') AS `a`, isnull(`b`, 'Short Term') AS `b`
FROM t0`

Let me know if anything is wrong in my understanding.


Discussion:

Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -1,10 +1,11 @@
-from operator import methodcaller
+import operator
 
 import toolz
 
 import ibis.expr.lineage as lin
 import ibis.expr.operations as ops
 import ibis.expr.types as ir
+from ibis import util
 from ibis.common import ExpressionError, IbisTypeError, RelationError
 from ibis.expr.schema import HasSchema
 from ibis.expr.window import window
@@ -290,35 +291,36 @@ def get_result(self):
         return result
 
     def _lift_arg(self, arg, block=None):
-        unchanged = [True]
+        changed = 0
 
-        def _lift(x):
-            if isinstance(x, ir.Expr):
-                lifted_arg = self.lift(x, block=block)
-                if lifted_arg is not x:
-                    unchanged[0] = False
+        def _lift(expr):
+            nonlocal changed
+
+            if isinstance(expr, ir.Expr):
+                lifted_arg = self.lift(expr, block=block)
+                changed += lifted_arg is not expr
             else:
                 # a string or some other thing
-                lifted_arg = x
+                lifted_arg = expr
             return lifted_arg
 
         if arg is None:
             return arg, True
 
-        if isinstance(arg, (tuple, list)):
-            result = [_lift(x) for x in arg]
+        if util.is_iterable(arg):
+            result = list(map(_lift, arg))
         else:
             result = _lift(arg)
 
-        return result, unchanged[0]
+        return result, not changed
 
     def lift(self, expr, block=None):
-        key = expr.op(), block
+        op, _ = key = expr.op(), block
 
-        if key in self.lift_memo:
+        try:
             return self.lift_memo[key]
-
-        op = expr.op()
+        except KeyError:
+            pass
 
         if isinstance(op, ops.ValueOp):
             return self._sub(expr, block=block)
@@ -343,26 +345,30 @@ def _lift_TableColumn(self, expr, block=None):
 
         if isinstance(root, ops.Selection):
             can_lift = False
+            all_simple_columns = all(
+                isinstance(sel.op(), ops.TableColumn)
+                and sel.op().name == sel.get_name()
+                for sel in root.selections
+                if isinstance(sel, ir.ValueExpr)
+                if sel.has_name()
+            )
 
             for val in root.selections:
+                value_op = val.op()
                 if (
-                    isinstance(val.op(), ops.PhysicalTable)
+                    isinstance(value_op, ops.PhysicalTable)
                     and node.name in val.schema()
                 ):
                     can_lift = True
                     lifted_root = self.lift(val)
                 elif (
-                    isinstance(val.op(), ops.TableColumn)
-                    and val.op().name == val.get_name()
+                    all_simple_columns
+                    and isinstance(val, ir.ValueExpr)
+                    and val.has_name()
                     and node.name == val.get_name()
                 ):
                     can_lift = True
-                    lifted_root = self.lift(val.op().table)
-
-            # HACK: If we've projected a join, do not lift the children
-            # TODO: what about limits and other things?
-            # if isinstance(root.table.op(), Join):
-            #     can_lift = False
+                    lifted_root = self.lift(value_op.table)
 
             if can_lift and not block:
                 lifted_node = ops.TableColumn(node.name, lifted_root)
@@ -375,14 +381,15 @@ def _lift_Aggregation(self, expr, block=None):
             block = self.block_projection
 
         op = expr.op()
+        table = op.table
 
         # as exposed in #544, do not lift the table inside (which may be
         # filtered or otherwise altered in some way) if blocking
 
         if block:
-            lifted_table = op.table
+            lifted_table = table
         else:
-            lifted_table = self.lift(op.table, block=True)
+            lifted_table = self.lift(table, block=True)
 
         unch = lifted_table is op.table
 
@@ -396,7 +403,7 @@ def _lift_Aggregation(self, expr, block=None):
             lifted_op = ops.Aggregation(
                 lifted_table, lifted_aggs, by=lifted_by, having=lifted_having
             )
-            result = ir.TableExpr(lifted_op)
+            result = lifted_op.to_expr()
         else:
             result = expr
 
@@ -430,7 +437,7 @@ def _lift_Selection(self, expr, block=None):
                 lifted_predicates,
                 lifted_sort_keys,
             )
-            result = ir.TableExpr(lifted_projection)
+            result = lifted_projection.to_expr()
         else:
             result = expr
 
@@ -771,15 +778,19 @@ def try_fusion(self, root):
         validator = ExprValidator([root_table])
         fused_exprs = []
         can_fuse = False
+        clean_exprs = self.clean_exprs
 
         if not isinstance(root_table.op(), ops.Join):
-            resolved = _maybe_resolve_exprs(root_table, self.input_exprs)
+            try:
+                resolved = root_table._resolve(self.input_exprs)
+            except (AttributeError, IbisTypeError):
+                resolved = clean_exprs
         else:
             # joins cannot be used to resolve expressions, but we still may be
             # able to fuse columns from a projection off of a join. In that
             # case, use the projection's input expressions as the columns with
             # which to attempt fusion
-            resolved = self.clean_exprs
+            resolved = clean_exprs
 
         if not resolved:
             return None
@@ -830,13 +841,6 @@ def try_fusion(self, root):
         return None
 
 
-def _maybe_resolve_exprs(table, exprs):
-    try:
-        return table._resolve(exprs)
-    except (AttributeError, IbisTypeError):
-        return None
-
-
 class ExprValidator:
     def __init__(self, exprs):
         self.parent_exprs = exprs
@@ -1020,7 +1024,7 @@ def finder(expr):
             return lin.proceed, None
 
     first_tables = lin.traverse(finder, expr.op().flat_args())
-    options = list(toolz.unique(first_tables, key=methodcaller('op')))
+    options = list(toolz.unique(first_tables, key=operator.methodcaller('op')))
 
     if len(options) > 1:
         raise NotImplementedError('More than one base table not implemented')
diff --git a/ibis/expr/api.py b/ibis/expr/api.py
--- a/ibis/expr/api.py
+++ b/ibis/expr/api.py
@@ -3360,43 +3360,29 @@ def mutate(table, exprs=None, **mutations):
     >>> expr2 = table.mutate(new_columns)
     >>> expr.equals(expr2)
     True
+
     """
-    if exprs is None:
-        exprs = []
-    else:
-        exprs = util.promote_list(exprs)
+    exprs = [] if exprs is None else util.promote_list(exprs)
+    exprs.extend(
+        (expr(table) if util.is_function(expr) else as_value_expr(expr)).name(
+            name
+        )
+        for name, expr in sorted(mutations.items(), key=operator.itemgetter(0))
+    )
 
-    for k, v in sorted(mutations.items(), key=operator.itemgetter(0)):
-        if util.is_function(v):
-            v = v(table)
-        else:
-            v = as_value_expr(v)
-
-        exprs.append(v.name(k))
-
-    has_replacement = False
-    for expr in exprs:
-        if expr.get_name() in table:
-            has_replacement = True
-
-    if has_replacement:
-        by_name = dict((x.get_name(), x) for x in exprs)
-        used = set()
-        proj_exprs = []
-        for c in table.columns:
-            if c in by_name:
-                proj_exprs.append(by_name[c])
-                used.add(c)
-            else:
-                proj_exprs.append(c)
-
-        for x in exprs:
-            if x.get_name() not in used:
-                proj_exprs.append(x)
-
-        return table.projection(proj_exprs)
+    by_name = collections.OrderedDict(
+        (expr.get_name(), expr) for expr in exprs
+    )
+    columns = table.columns
+    used = by_name.keys() & columns
+
+    if used:
+        proj_exprs = [
+            by_name.get(column, table[column]) for column in columns
+        ] + [expr for name, expr in by_name.items() if name not in used]
     else:
-        return table.projection([table] + exprs)
+        proj_exprs = [table] + exprs
+    return table.projection(proj_exprs)
 
 
 def projection(table, exprs):
@@ -3497,7 +3483,6 @@ def projection(table, exprs):
         exprs = [exprs]
 
     projector = L.Projector(table, exprs)
-
     op = projector.get_result()
     return op.to_expr()
 
diff --git a/ibis/expr/types.py b/ibis/expr/types.py
--- a/ibis/expr/types.py
+++ b/ibis/expr/types.py
@@ -2,6 +2,8 @@
 import os
 import webbrowser
 
+import numpy as np
+
 import ibis
 import ibis.common as com
 import ibis.config as config
@@ -177,10 +179,7 @@ def op(self):
 
     @property
     def _factory(self):
-        def factory(arg, name=None):
-            return type(self)(arg, name=name)
-
-        return factory
+        return type(self)
 
     def execute(self, limit='default', params=None, **kwargs):
         """
@@ -472,18 +471,12 @@ def __dir__(self):
 
     def _resolve(self, exprs):
         exprs = util.promote_list(exprs)
-
-        # Stash this helper method here for now
-        out_exprs = []
-        for expr in exprs:
-            expr = self._ensure_expr(expr)
-            out_exprs.append(expr)
-        return out_exprs
+        return list(map(self._ensure_expr, exprs))
 
     def _ensure_expr(self, expr):
         if isinstance(expr, str):
             return self[expr]
-        elif isinstance(expr, int):
+        elif isinstance(expr, (int, np.integer)):
             return self[self.schema().name_at_position(expr)]
         elif not isinstance(expr, Expr):
             return expr(self)


Test patch:
diff --git a/ibis/expr/tests/test_analysis.py b/ibis/expr/tests/test_analysis.py
--- a/ibis/expr/tests/test_analysis.py
+++ b/ibis/expr/tests/test_analysis.py
@@ -56,7 +56,8 @@ def test_rewrite_past_projection(con):
     table5 = table[(table.f * 2).name('c'), table.f]
     expr = table5['c'] == 2
     result = L.substitute_parents(expr)
-    assert result is expr
+    expected = expr
+    assert result.equals(expected)
 
 
 def test_multiple_join_deeper_reference():
@@ -207,14 +208,11 @@ def test_fuse_filter_sort_by():
 
 def test_no_rewrite(con):
     table = con.table('test1')
-
-    # Substitution not fully possible if we depend on a new expr in a
-    # projection
     table4 = table[['c', (table['c'] * 2).name('foo')]]
     expr = table4['c'] == table4['foo']
     result = L.substitute_parents(expr)
-    expected = table['c'] == table4['foo']
-    assert_equal(result, expected)
+    expected = expr
+    assert result.equals(expected)
 
 
 # def test_projection_with_join_pushdown_rewrite_refs():
diff --git a/ibis/expr/tests/test_table.py b/ibis/expr/tests/test_table.py
--- a/ibis/expr/tests/test_table.py
+++ b/ibis/expr/tests/test_table.py
@@ -18,7 +18,7 @@
 
 def test_empty_schema():
     table = api.table([], 'foo')
-    assert len(table.schema()) == 0
+    assert not table.schema()
 
 
 def test_columns(con):
@@ -1220,3 +1220,15 @@ def test_unbound_table_name():
     name = t.op().name
     match = re.match(r'^unbound_table_\d+$', name)
     assert match is not None
+
+
+def test_mutate_chain():
+    one = ibis.table([('a', 'string'), ('b', 'string')], name='t')
+    two = one.mutate(b=lambda t: t.b.fillna('Short Term'))
+    three = two.mutate(a=lambda t: t.a.fillna('Short Term'))
+    a, b = three.op().selections
+
+    # we can't fuse these correctly yet
+    assert isinstance(a.op(), ops.IfNull)
+    assert isinstance(b.op(), ops.TableColumn)
+    assert isinstance(b.op().table.op().selections[1].op(), ops.IfNull)
diff --git a/ibis/tests/backends.py b/ibis/tests/backends.py
--- a/ibis/tests/backends.py
+++ b/ibis/tests/backends.py
@@ -308,7 +308,7 @@ def functional_alltypes(self):
         return t.mutate(bool_col=t.bool_col == 1)
 
 
-class Clickhouse(Backend, RoundHalfToEven):
+class Clickhouse(UnorderedComparator, Backend, RoundHalfToEven):
     check_dtype = False
     supports_window_operations = False
     returned_timestamp_unit = 's'