Repository: pandas-dev/pandas

Issue:
regression: NotImplementedError: operator '*' not implemented for bool dtypes
seemed to work just fine (with  0.13.1) and I do not see why it should not if you are decorating numpy arrays here:

```
  File "/home/neurodebian/proj/numpy-vbench/vbench/vbench/analysis.py", line 83, in __call__
    Tts, Tps = ss.ttest_1samp(test_samples.timing, results.timing)
  File "/usr/lib/python2.7/dist-packages/scipy/stats/stats.py", line 2887, in ttest_1samp
    t = np.where((d==0)*(v==0), 1.0, t) #define t=0/0 = 1, identical mean, var
  File "/home/neurodebian/deb/gits/pkg-exppsy/pandas/pandas/core/ops.py", line 505, in wrapper
    return left._constructor(wrap_results(na_op(lvalues, rvalues)),
  File "/home/neurodebian/deb/gits/pkg-exppsy/pandas/pandas/core/ops.py", line 445, in na_op
    raise_on_error=True, **eval_kwargs)
  File "/home/neurodebian/deb/gits/pkg-exppsy/pandas/pandas/computation/expressions.py", line 196, in evaluate
    _bool_arith_check(op_str, a, b)
  File "/home/neurodebian/deb/gits/pkg-exppsy/pandas/pandas/computation/expressions.py", line 177, in _bool_arith_check
    "dtypes" % op_str)
NotImplementedError: operator '*' not implemented for bool dtypes
```



Discussion:
on purpose `numexpr` doesn't allow this and since that is heavily used are banning it.

should be using `&` for anding

discussions here: #7011, #6762, #7015 

Could that be added in the error message? (that you can use `&` instead)

sure...I don't think that's hard, FYI I don't think their is an analog for `|`, is there?

@cpcloud can you change that error message a bit?

Dure

I mean sure

oh well -- just don't get surprised then whenever people with stock scipy would complain that they can't even do a ttest now (whenever it was working before)... FWIW -- scipy has refactored that code I believe in v0.11.0b1~20^2, so only wheezy and precise (among sensible)  would be affected.

"sure...I don't think that's hard, FYI I don't think their is an analog for |, is there?"

Isn't that simply `+`? 

its not `+`

`True | True == True`
`True + True == 2`

aha but:

```
In [2]: np.array([True]) + np.array([True])
Out[2]: array([ True], dtype=bool)
```

interesting. 

@cpcloud another possibility is to try to detect these condition and then just do the eval in python space (which will work I guess), maybe with a warning?

i pretty much do that now ... but i raise instead of continuing on

so i'll change it

I had the dtype=bool behaviour in mind, but even apart from that, while `True != 2`, `bool(True) == bool(2)` which (as we say here) is close enough for government work.

What do we do with this one? Ideally this would be handled for 0.14 (as this is introduced in 0.14)

@jorisvandenbossche agreed i'll have something up tonight

maybe easiest to do a translation, warning then evaluate it?

yep something like:

```
if bool arith op: warn("don't do this") -> do_it_anyway()
```

@cpcloud PR for this?

Yep sorry got a bit sidetracked practicing my talk. :) should have something by lunch. 

so really this should just be `*`, `+` and `-`, since none of the others really make sense

yep

interesting never really figured out what `x - y` for bool does, it's actually `xor`:

```
In [58]: x, y = rand(10) > 0.5, rand(10) > 0.5

In [59]: np.all((x | y) & ~(x & y) == x - y)
Out[59]: True
```

`x-y == x^y` as well (which is the `xor` operator)

ah right ... thanks that helps actually (simplifies the error message)


Resolving patch:
diff --git a/pandas/computation/expressions.py b/pandas/computation/expressions.py
--- a/pandas/computation/expressions.py
+++ b/pandas/computation/expressions.py
@@ -6,6 +6,7 @@
 
 """
 
+import warnings
 import numpy as np
 from pandas.core.common import _values_from_object
 from distutils.version import LooseVersion
@@ -170,11 +171,23 @@ def _has_bool_dtype(x):
             return isinstance(x, (bool, np.bool_))
 
 
-def _bool_arith_check(op_str, a, b, not_allowed=frozenset(('+', '*', '-', '/',
-                                                           '//', '**'))):
-    if op_str in not_allowed and _has_bool_dtype(a) and _has_bool_dtype(b):
-        raise NotImplementedError("operator %r not implemented for bool "
-                                  "dtypes" % op_str)
+def _bool_arith_check(op_str, a, b, not_allowed=frozenset(('/', '//', '**')),
+                      unsupported=None):
+    if unsupported is None:
+        unsupported = {'+': '|', '*': '&', '-': '^'}
+
+    if _has_bool_dtype(a) and _has_bool_dtype(b):
+        if op_str in unsupported:
+            warnings.warn("evaluating in Python space because the %r operator"
+                          " is not supported by numexpr for the bool "
+                          "dtype, use %r instead" % (op_str,
+                                                     unsupported[op_str]))
+            return False
+
+        if op_str in not_allowed:
+            raise NotImplementedError("operator %r not implemented for bool "
+                                      "dtypes" % op_str)
+    return True
 
 
 def evaluate(op, op_str, a, b, raise_on_error=False, use_numexpr=True,
@@ -193,7 +206,7 @@ def evaluate(op, op_str, a, b, raise_on_error=False, use_numexpr=True,
                          return the results
         use_numexpr : whether to try to use numexpr (default True)
         """
-    _bool_arith_check(op_str, a, b)
+    use_numexpr = use_numexpr and _bool_arith_check(op_str, a, b)
     if use_numexpr:
         return _evaluate(op, op_str, a, b, raise_on_error=raise_on_error,
                          **eval_kwargs)


Test patch:
diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py
--- a/pandas/tests/test_expressions.py
+++ b/pandas/tests/test_expressions.py
@@ -343,8 +343,8 @@ def testit():
     def test_bool_ops_raise_on_arithmetic(self):
         df = DataFrame({'a': np.random.rand(10) > 0.5,
                         'b': np.random.rand(10) > 0.5})
-        names = 'add', 'mul', 'sub', 'div', 'truediv', 'floordiv', 'pow'
-        ops = '+', '*', '-', '/', '/', '//', '**'
+        names = 'div', 'truediv', 'floordiv', 'pow'
+        ops = '/', '/', '//', '**'
         msg = 'operator %r not implemented for bool dtypes'
         for op, name in zip(ops, names):
             if not compat.PY3 or name != 'div':
@@ -369,6 +369,49 @@ def test_bool_ops_raise_on_arithmetic(self):
                 with tm.assertRaisesRegexp(TypeError, err_msg):
                     f(df, True)
 
+    def test_bool_ops_warn_on_arithmetic(self):
+        n = 10
+        df = DataFrame({'a': np.random.rand(n) > 0.5,
+                        'b': np.random.rand(n) > 0.5})
+        names = 'add', 'mul', 'sub'
+        ops = '+', '*', '-'
+        subs = {'+': '|', '*': '&', '-': '^'}
+        sub_funcs = {'|': 'or_', '&': 'and_', '^': 'xor'}
+        for op, name in zip(ops, names):
+            f = getattr(operator, name)
+            fe = getattr(operator, sub_funcs[subs[op]])
+
+            with tm.use_numexpr(True, min_elements=5):
+                with tm.assert_produces_warning():
+                    r = f(df, df)
+                    e = fe(df, df)
+                    tm.assert_frame_equal(r, e)
+
+                with tm.assert_produces_warning():
+                    r = f(df.a, df.b)
+                    e = fe(df.a, df.b)
+                    tm.assert_series_equal(r, e)
+
+                with tm.assert_produces_warning():
+                    r = f(df.a, True)
+                    e = fe(df.a, True)
+                    tm.assert_series_equal(r, e)
+
+                with tm.assert_produces_warning():
+                    r = f(False, df.a)
+                    e = fe(False, df.a)
+                    tm.assert_series_equal(r, e)
+
+                with tm.assert_produces_warning():
+                    r = f(False, df)
+                    e = fe(False, df)
+                    tm.assert_frame_equal(r, e)
+
+                with tm.assert_produces_warning():
+                    r = f(df, True)
+                    e = fe(df, True)
+                    tm.assert_frame_equal(r, e)
+
 
 if __name__ == '__main__':
     import nose
diff --git a/pandas/util/testing.py b/pandas/util/testing.py
--- a/pandas/util/testing.py
+++ b/pandas/util/testing.py
@@ -35,6 +35,8 @@
     raise_with_traceback, httplib
 )
 
+from pandas.computation import expressions as expr
+
 from pandas import bdate_range
 from pandas.tseries.index import DatetimeIndex
 from pandas.tseries.period import PeriodIndex
@@ -1576,3 +1578,14 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
 
         np.random.set_state(self.start_state)
+
+
+@contextmanager
+def use_numexpr(use, min_elements=expr._MIN_ELEMENTS):
+    olduse = expr._USE_NUMEXPR
+    oldmin = expr._MIN_ELEMENTS
+    expr.set_use_numexpr(use)
+    expr._MIN_ELEMENTS = min_elements
+    yield
+    expr._MIN_ELEMENTS = oldmin
+    expr.set_use_numexpr(olduse)