Repository: blaze/blaze

Issue:
Unexpected exceptions when combining interactive and non-interactive symbols
Exception when combining an `InteractiveSymbol` with a `Symbol`:

``` python
In [25]: import blaze as bz

In [26]: y = bz.Symbol('y', 'float64')

In [29]: iris = bz.Data('./blaze/examples/data/iris.csv')

In [30]: iris.sepal_length / y
Out[30]: ---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/Users/ksmith/anaconda/lib/python2.7/site-packages/IPython/core/formatters.pyc in __call__(self, obj)
    695                 type_pprinters=self.type_printers,
    696                 deferred_pprinters=self.deferred_printers)
--> 697             printer.pretty(obj)
    698             printer.flush()
    699             return stream.getvalue()

/Users/ksmith/anaconda/lib/python2.7/site-packages/IPython/lib/pretty.pyc in pretty(self, obj)
    381                             if callable(meth):
    382                                 return meth(obj, self, cycle)
--> 383             return _default_pprint(obj, self, cycle)
    384         finally:
    385             self.end_group()

/Users/ksmith/anaconda/lib/python2.7/site-packages/IPython/lib/pretty.pyc in _default_pprint(obj, p, cycle)
    501     if _safe_getattr(klass, '__repr__', None) not in _baseclass_reprs:
    502         # A user-provided repr. Find newlines and replace them with p.break_()
--> 503         _repr_pprint(obj, p, cycle)
    504         return
    505     p.begin_group(1, '<')

/Users/ksmith/anaconda/lib/python2.7/site-packages/IPython/lib/pretty.pyc in _repr_pprint(obj, p, cycle)
    683     """A pprint that just redirects to the normal repr function."""
    684     # Find newlines and replace them with p.break_()
--> 685     output = repr(obj)
    686     for idx,output_line in enumerate(output.splitlines()):
    687         if idx:

/Users/ksmith/work/blaze/blaze-repo/blaze/interactive.pyc in expr_repr(expr, n)
    267                              isscalar(expr.dshape.measure) or
    268                              isinstance(expr.dshape.measure, Map))):
--> 269         return repr_tables(expr, 10)
    270
    271     # Smallish arrays

/Users/ksmith/work/blaze/blaze-repo/blaze/interactive.pyc in repr_tables(expr, n)
    203
    204 def repr_tables(expr, n=10):
--> 205     result = concrete_head(expr, n).rename(columns={None: ''})
    206
    207     if isinstance(result, (DataFrame, Series)):

/Users/ksmith/work/blaze/blaze-repo/blaze/interactive.pyc in concrete_head(expr, n)
    187         return odo(head, DataFrame)
    188     else:
--> 189         df = odo(head, DataFrame)
    190         df.columns = [expr._name]
    191         return df

/Users/ksmith/work/odo/odo/odo.pyc in odo(source, target, **kwargs)
     89     odo.append.append      - Add things onto existing things
     90     """
---> 91     return into(target, source, **kwargs)

/Users/ksmith/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.pyc in __call__(self, *args, **kwargs)
    162             self._cache[types] = func
    163         try:
--> 164             return func(*args, **kwargs)
    165
    166         except MDNotImplementedError:

/Users/ksmith/work/blaze/blaze-repo/blaze/interactive.pyc in into(a, b, **kwargs)
    311 @dispatch((object, type, str, unicode), Expr)
    312 def into(a, b, **kwargs):
--> 313     result = compute(b, **kwargs)
    314     kwargs['dshape'] = b.dshape
    315     return into(a, result, **kwargs)

/Users/ksmith/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.pyc in __call__(self, *args, **kwargs)
    162             self._cache[types] = func
    163         try:
--> 164             return func(*args, **kwargs)
    165
    166         except MDNotImplementedError:

/Users/ksmith/work/blaze/blaze-repo/blaze/interactive.pyc in compute(expr, **kwargs)
    170         raise ValueError("No data resources found")
    171     else:
--> 172         return compute(expr, resources, **kwargs)
    173
    174

/Users/ksmith/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.pyc in __call__(self, *args, **kwargs)
    162             self._cache[types] = func
    163         try:
--> 164             return func(*args, **kwargs)
    165
    166         except MDNotImplementedError:

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in compute(expr, d, **kwargs)
    401         d4 = d3
    402
--> 403     result = top_then_bottom_then_top_again_etc(expr3, d4, **kwargs)
    404     if post_compute_:
    405         result = post_compute_(expr3, result, scope=d4)

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in top_then_bottom_then_top_again_etc(expr, scope, **kwargs)
    165
    166     # 2. Compute from the bottom until there is a data type change
--> 167     expr2, scope2 = bottom_up_until_type_break(expr, scope, **kwargs)
    168
    169     # 3. Re-optimize data and expressions

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in bottom_up_until_type_break(expr, scope, **kwargs)
    306     #    (this is the bottom part of bottom up)
    307     exprs, new_scopes = zip(*[bottom_up_until_type_break(i, scope, **kwargs)
--> 308                               for i in inputs])
    309
    310     # 2. Form new (much shallower) expression and new (more computed) scope

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in bottom_up_until_type_break(expr, scope, **kwargs)
    306     #    (this is the bottom part of bottom up)
    307     exprs, new_scopes = zip(*[bottom_up_until_type_break(i, scope, **kwargs)
--> 308                               for i in inputs])
    309
    310     # 2. Form new (much shallower) expression and new (more computed) scope

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in bottom_up_until_type_break(expr, scope, **kwargs)
    306     #    (this is the bottom part of bottom up)
    307     exprs, new_scopes = zip(*[bottom_up_until_type_break(i, scope, **kwargs)
--> 308                               for i in inputs])
    309
    310     # 2. Form new (much shallower) expression and new (more computed) scope

ValueError: need more than 0 values to unpack
```

What should be the result here?  To my mind, it should be another expression that's partially bound--`iris` is a bound symbol, `y` is unbound.  It can't be evaluated because `y` is unbound.

Getting an exception here is unexpected.

This works:

``` python
In [34]: bz.compute(iris.sepal_length / y, {y: 2.})
Out[34]:
0      2.55
1      2.45
2      2.35
3      2.30
4      2.50
5      2.70
...
Name: sepal_length, dtype: float64
```

But this doesn't:

``` python
In [35]: bz.compute(bz.transform(iris, ratio=iris.sepal_length / y), {y: 2})
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-35-26dd478b38c1> in <module>()
----> 1 bz.compute(bz.transform(iris, ratio=iris.sepal_length / y), {y: 2})

/Users/ksmith/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.pyc in __call__(self, *args, **kwargs)
    162             self._cache[types] = func
    163         try:
--> 164             return func(*args, **kwargs)
    165
    166         except MDNotImplementedError:

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in compute(expr, d, **kwargs)
    401         d4 = d3
    402
--> 403     result = top_then_bottom_then_top_again_etc(expr3, d4, **kwargs)
    404     if post_compute_:
    405         result = post_compute_(expr3, result, scope=d4)

/Users/ksmith/work/blaze/blaze-repo/blaze/compute/core.pyc in top_then_bottom_then_top_again_etc(expr, scope, **kwargs)
    179         try:
    180             expr3 = optimize_(expr2, *[scope3[leaf]
--> 181                                        for leaf in expr2._leaves()])
    182             _d = dict(zip(expr2._leaves(), expr3._leaves()))
    183             scope4 = dict((e._subs(_d), d) for e, d in scope3.items())

KeyError: y
```



Discussion:
What a coincidence, I was just submitting a fix for this. I think this patch is all that's needed:

``` diff
--- a/blaze/interactive.py
+++ b/blaze/interactive.py
@@ -255,7 +255,7 @@ def coerce_scalar(result, dshape, odo_kwargs=None):

 def expr_repr(expr, n=10):
     # Pure Expressions, not interactive
-    if not expr._resources():
+    if not set(expr._resources().keys()).issuperset(expr._leaves()):
         return str(expr)
```

This makes sense. Should we also somehow indicate in the repr that some data is bound?

you can use six.viewkeys here instead of set(x.keys()). This will call the dict.viewkeys() in py2 or dict.keys() in py3

I was thinking of making `InteractiveSymbol.__str__` more informative. Perhaps:

``` python
def __str__(self):
    obj = object.__repr__(self._data)
    name = ", name='{0}'".format(self._name) if self._name else ""
    return "Data('{0}'{1})".format(obj, name)
```

I am not sure if we would want to repr the data. It might get kind of long making expressions harder to read. Also, they could contain a bunch of newlines like DataFrame.

That's why we use `object.__repr__` instead of `repr`. Leads to things like `'<pandas.core.frame.DataFrame object at 0x110ce02d0>'`. Still not very short though.

Ah, I didn't realize that. I am okay with that then since it won't blow up the strings too much. Should we open a seperate issue for this?

An alternative would be to go to a more verbose multiline model. Output something like:

``` python
In [12]: expr
Out[12]: _1[_1.a > a]

where:
    _1 = Data('<pandas.core.frame.DataFrame object at 0x110ce02d0>')
```

This might be nicer in expressions where the defined data is used multiple times. Idk.

A new issue might be best. I'll open one.

ooh, I really like the where block. This would  really help keep the expression clean while showing the imporant leaf values.


Resolving patch:
diff --git a/blaze/interactive.py b/blaze/interactive.py
--- a/blaze/interactive.py
+++ b/blaze/interactive.py
@@ -255,7 +255,7 @@ def coerce_scalar(result, dshape, odo_kwargs=None):
 
 def expr_repr(expr, n=10):
     # Pure Expressions, not interactive
-    if not expr._resources():
+    if not set(expr._resources().keys()).issuperset(expr._leaves()):
         return str(expr)
 
     # Scalars


Test patch:
diff --git a/blaze/tests/test_interactive.py b/blaze/tests/test_interactive.py
--- a/blaze/tests/test_interactive.py
+++ b/blaze/tests/test_interactive.py
@@ -459,3 +459,16 @@ def test_pickle_roundtrip():
 def test_nameless_data():
     data = [('a', 1)]
     assert repr(data) in repr(Data(data))
+
+
+def test_partially_bound_expr():
+    df = pd.DataFrame([(1, 'Alice', 100),
+                       (2, 'Bob', -200),
+                       (3, 'Charlie', 300),
+                       (4, 'Denis', 400),
+                       (5, 'Edith', -500)],
+                      columns=['id', 'name', 'balance'])
+    data = Data(df, name='data')
+    a = symbol('a', 'int')
+    expr = data.name[data.balance > a]
+    assert repr(expr) == 'data[data.balance > a].name'