Repository: blaze/blaze

Issue:
Numba type error
A failing example that generates a numba dtype error.  Looks like the types are misaligned.

``` Python
In [1]: from blaze import *
x
In [2]: x = np.array([[(4, 180), (4, 184), (4, 188), (4, 192), (4, 196)],
       [(4, 660), (4, 664), (4, 668), (4, 672), (4, 676)],
       [(4, 1140), (4, 1144), (4, 1148), (4, 1152), (4, 1156)],
       [(4, 1620), (4, 1624), (4, 1628), (4, 1632), (4, 1636)],
       [(4, 2100), (4, 2104), (4, 2108), (4, 2112), (4, 2116)]], 
      dtype=[('count', '<i4'), ('total', '<i8')])

In [3]: aggregate = Data(x)

In [4]: compute((sum(aggregate.total, axis=(0,))) / (sum(aggregate.count, axis=(0,))))
TypeError: ufunc 'ufunc' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''
> /home/mrocklin/workspace/blaze/blaze/compute/numba.py(208)broadcast_numba()
    206         return compute(t, dict(zip(t._leaves(), data)))
    207     else:
--> 208         return ufunc(*data)
ipdb> print ufunc.types 
['il->d']
ipdb> print data
(array([20, 20, 20, 20, 20]), array([5700, 5720, 5740, 5760, 5780]))
```

cc @cpcloud 



Discussion:
@mrocklin any reason you have a nested structure here?

```
ipdb> x
array([[(4, 180), (4, 184), (4, 188), (4, 192), (4, 196)],
       [(4, 660), (4, 664), (4, 668), (4, 672), (4, 676)],
       [(4, 1140), (4, 1144), (4, 1148), (4, 1152), (4, 1156)],
       [(4, 1620), (4, 1624), (4, 1628), (4, 1632), (4, 1636)],
       [(4, 2100), (4, 2104), (4, 2108), (4, 2112), (4, 2116)]],
      dtype=[('count', '<i4'), ('total', '<i8')])
ipdb> x.shape
(5, 5)
```

actually nevermind, it doesn't really matter, it should work with this anyway, fix coming soon

This came out of a chunked `foo.mean()` call.


Resolving patch:
diff --git a/blaze/compute/numpy.py b/blaze/compute/numpy.py
--- a/blaze/compute/numpy.py
+++ b/blaze/compute/numpy.py
@@ -12,6 +12,7 @@
 from ..expr import BinOp, UnaryOp, USub, Not, nelements
 from ..expr import UTCFromTimestamp, DateTimeTruncate
 from ..expr import Transpose, TensorDot
+from ..utils import keywords
 
 from .core import base, compute
 from ..dispatch import dispatch
@@ -88,14 +89,17 @@ def compute_up(t, x, **kwargs):
 
 @dispatch(count, np.ndarray)
 def compute_up(t, x, **kwargs):
+    result_dtype = to_numpy_dtype(t.dshape)
     if issubclass(x.dtype.type, (np.floating, np.object_)):
-        return pd.notnull(x).sum(keepdims=t.keepdims, axis=t.axis)
+        return pd.notnull(x).sum(keepdims=t.keepdims, axis=t.axis,
+                                 dtype=result_dtype)
     elif issubclass(x.dtype.type, np.datetime64):
-        return (x.view('int64') != inat).sum(keepdims=t.keepdims, axis=t.axis)
+        return (x.view('int64') != inat).sum(keepdims=t.keepdims, axis=t.axis,
+                                             dtype=result_dtype)
     else:
-        return np.ones(x.shape,
-                       dtype=to_numpy_dtype(t.dshape)).sum(keepdims=t.keepdims,
-                                                           axis=t.axis)
+        return np.ones(x.shape, dtype=result_dtype).sum(keepdims=t.keepdims,
+                                                        axis=t.axis,
+                                                        dtype=result_dtype)
 
 
 @dispatch(nunique, np.ndarray)
@@ -109,7 +113,12 @@ def compute_up(t, x, **kwargs):
 
 @dispatch(Reduction, np.ndarray)
 def compute_up(t, x, **kwargs):
-    return getattr(x, t.symbol)(axis=t.axis, keepdims=t.keepdims)
+    # can't use the method here, as they aren't Python functions
+    reducer = getattr(np, t.symbol)
+    if 'dtype' in keywords(reducer):
+        return reducer(x, axis=t.axis, keepdims=t.keepdims,
+                       dtype=to_numpy_dtype(t.schema))
+    return reducer(x, axis=t.axis, keepdims=t.keepdims)
 
 
 def axify(expr, axis, keepdims=False):
diff --git a/blaze/expr/reductions.py b/blaze/expr/reductions.py
--- a/blaze/expr/reductions.py
+++ b/blaze/expr/reductions.py
@@ -3,6 +3,7 @@
 import toolz
 from datashape import Record, DataShape, dshape
 from datashape import coretypes as ct
+import datashape
 
 from .core import common_subexpression
 from .expressions import Expr, symbol, ndim, Slice
@@ -62,7 +63,7 @@ def schema(self):
             result = toolz.first(schema.types)
         else:
             result = schema
-        return DataShape(schema)
+        return DataShape(result)
 
     @property
     def symbol(self):
@@ -100,7 +101,9 @@ class all(Reduction):
 
 
 class sum(Reduction):
-    pass
+    @property
+    def schema(self):
+        return DataShape(datashape.maxtype(super(sum, self).schema))
 
 
 class max(Reduction):
@@ -167,11 +170,11 @@ def __init__(self, child, unbiased=False, *args, **kwargs):
 
 class count(Reduction):
     """ The number of non-null elements """
-    schema = dshape(ct.int_)
+    schema = dshape(ct.int32)
 
 
 class nunique(Reduction):
-    schema = dshape(ct.int_)
+    schema = dshape(ct.int32)
 
 
 class nelements(Reduction):
@@ -188,7 +191,7 @@ class nelements(Reduction):
     >>> t[t.amount < 1].nelements()
     nelements(t[t.amount < 1])
     """
-    schema = dshape(ct.int_)
+    schema = dshape(ct.int32)
 
 
 def nrows(expr):


Test patch:
diff --git a/blaze/compute/tests/test_core_compute.py b/blaze/compute/tests/test_core_compute.py
--- a/blaze/compute/tests/test_core_compute.py
+++ b/blaze/compute/tests/test_core_compute.py
@@ -60,7 +60,7 @@ def test_bottom_up_until_type_break():
 
     e = s.amount.sum() + 1
     expr, scope = bottom_up_until_type_break(e, {s: data})
-    amount_sum = symbol('amount_sum', 'int')
+    amount_sum = symbol('amount_sum', 'int64')
     assert expr.isidentical(amount_sum + 1)
     assert len(scope) == 1
     assert amount_sum in scope
@@ -77,7 +77,7 @@ def test_bottom_up_until_type_break():
 
 
 def test_top_then_bottom_then_top_again_etc():
-    s = symbol('s', 'var * {name: string, amount: int}')
+    s = symbol('s', 'var * {name: string, amount: int32}')
     data = np.array([('Alice', 100), ('Bob', 200), ('Charlie', 300)],
                     dtype=[('name', 'S7'), ('amount', 'i4')])
 
diff --git a/blaze/compute/tests/test_numpy_compute.py b/blaze/compute/tests/test_numpy_compute.py
--- a/blaze/compute/tests/test_numpy_compute.py
+++ b/blaze/compute/tests/test_numpy_compute.py
@@ -189,7 +189,7 @@ def test_summary_on_ndarray():
 
     result = compute(summary(total=a.sum(), min=a.min(), keepdims=True), ax)
     expected = np.array([(ax.min(), ax.sum())],
-                        dtype=[('min', 'f4'), ('total', 'f4')])
+                        dtype=[('min', 'float32'), ('total', 'float64')])
     assert result.ndim == ax.ndim
     assert eq(expected, result)
 
@@ -333,3 +333,18 @@ def test_subexpr_datetime():
     result = compute(s.truncate(days=2).day, data)
     expected = np.array([31, 2, 2, 4])
     np.testing.assert_array_equal(result, expected)
+
+
+def test_mixed_types():
+    x = np.array([[(4, 180), (4, 184), (4, 188), (4, 192), (4, 196)],
+                  [(4, 660), (4, 664), (4, 668), (4, 672), (4, 676)],
+                  [(4, 1140), (4, 1144), (4, 1148), (4, 1152), (4, 1156)],
+                  [(4, 1620), (4, 1624), (4, 1628), (4, 1632), (4, 1636)],
+                  [(4, 2100), (4, 2104), (4, 2108), (4, 2112), (4, 2116)]],
+                 dtype=[('count', '<i4'), ('total', '<i8')])
+    aggregate = symbol('aggregate', discover(x))
+    result = compute(aggregate.total.sum(axis=(0,)) /
+                     aggregate.count.sum(axis=(0,)), x)
+    expected = (x['total'].sum(axis=0, keepdims=True) /
+                x['count'].sum(axis=0, keepdims=True)).squeeze()
+    np.testing.assert_array_equal(result, expected)
diff --git a/blaze/expr/tests/test_reductions.py b/blaze/expr/tests/test_reductions.py
--- a/blaze/expr/tests/test_reductions.py
+++ b/blaze/expr/tests/test_reductions.py
@@ -4,17 +4,17 @@
 
 def test_reduction_dshape():
     x = symbol('x', '5 * 3 * float32')
-    assert x.sum().dshape == x.schema
-    assert x.sum(axis=0).dshape == dshape('3 * float32')
-    assert x.sum(axis=1).dshape == dshape('5 * float32')
-    assert x.sum(axis=(0, 1)).dshape == dshape('float32')
+    assert x.sum().dshape == dshape('float64')
+    assert x.sum(axis=0).dshape == dshape('3 * float64')
+    assert x.sum(axis=1).dshape == dshape('5 * float64')
+    assert x.sum(axis=(0, 1)).dshape == dshape('float64')
 
 
 def test_keepdims():
     x = symbol('x', '5 * 3 * float32')
-    assert x.sum(axis=0, keepdims=True).dshape == dshape('1 * 3 * float32')
-    assert x.sum(axis=1, keepdims=True).dshape == dshape('5 * 1 * float32')
-    assert x.sum(axis=(0, 1), keepdims=True).dshape == dshape('1 * 1 * float32')
+    assert x.sum(axis=0, keepdims=True).dshape == dshape('1 * 3 * float64')
+    assert x.sum(axis=1, keepdims=True).dshape == dshape('5 * 1 * float64')
+    assert x.sum(axis=(0, 1), keepdims=True).dshape == dshape('1 * 1 * float64')
 
     assert x.std(axis=0, keepdims=True).shape == (1, 3)
 
diff --git a/blaze/expr/tests/test_split.py b/blaze/expr/tests/test_split.py
--- a/blaze/expr/tests/test_split.py
+++ b/blaze/expr/tests/test_split.py
@@ -3,7 +3,7 @@
 from datashape import dshape
 from datashape.predicates import isscalar, isrecord, iscollection
 
-t = symbol('t', 'var * {name: string, amount: int, id: int}')
+t = symbol('t', 'var * {name: string, amount: int32, id: int32}')
 a = symbol('a', '1000 * 2000 * {x: float32, y: float32}')
 
 
@@ -283,7 +283,7 @@ def test_agg_shape_in_tabular_case_with_explicit_chunk():
     expr = by(t.name, total=t.amount.sum())
     (chunk, chunk_expr), (agg, agg_expr) = split(t, expr, chunk=c)
 
-    assert agg.dshape == dshape('var * {name: string, total: int}')
+    assert agg.dshape == dshape('var * {name: string, total: int64}')
 
 
 def test_reductions():
diff --git a/blaze/expr/tests/test_table.py b/blaze/expr/tests/test_table.py
--- a/blaze/expr/tests/test_table.py
+++ b/blaze/expr/tests/test_table.py
@@ -336,12 +336,9 @@ def test_unary_ops():
 def test_reduction():
     t = TableSymbol('t', '{name: string, amount: int32}')
     r = sum(t['amount'])
-    print(type(r.dshape))
-    print(type(dshape('int32')))
-    print(r.dshape)
-    assert r.dshape in (dshape('int32'),
-                        dshape('{amount: int32}'),
-                        dshape('{amount_sum: int32}'))
+    assert r.dshape in (dshape('int64'),
+                        dshape('{amount: int64}'),
+                        dshape('{amount_sum: int64}'))
 
     assert 'amount' not in str(t.count().dshape)
 
@@ -432,7 +429,7 @@ def test_scalar_uadd(self, symsum):
 def test_summary():
     t = TableSymbol('t', '{id: int32, name: string, amount: int32}')
     s = summary(total=t.amount.sum(), num=t.id.count())
-    assert s.dshape == dshape('{num: int32, total: int32}')
+    assert s.dshape == dshape('{num: int32, total: int64}')
     assert hash(s)
     assert eval(str(s)).isidentical(s)