Repository: pandas-dev/pandas

Issue:
BUG: replace should handle Bool blocks correctly
http://stackoverflow.com/questions/21733802/pandas-series-replace-on-boolean-series

by definition they don't have missing values



Discussion:
Are we ever going to get an integer or bool NaN?  I'm haven't followed any developments [in this](http://pandas.pydata.org/pandas-docs/dev/gotchas.html#support-for-integer-na).

its not _that_ difficult if  you use a `NaT` like value (e.g. `np.iinfo(np.int64).min)`) (which would have to be set for each of the dtypes. (and bool would have to be represented by say `np.uint8` or something. 

Then of course you would have to go thru lots of routines and deal with the special handling (that is implicity in that you _know_ that int64 if its a dtype of int64 cannot right now have a missing value).

so would have to have someone with lots of time to do this!

up for it @TomAugspurger ?

I wasn't volunteering :) Maybe someday. I was just asking because I was thinking about skipping `int` and `bool` blocks in the interpolation routines.

@TomAugspurger yep....that's makes sense for interpolate for sure

technically `replace` actually shouldn't ignore on `Bool` blocks but only replace if the values are `True/False` 

@cpcloud want 2 take this?

Happy to fix the bool issue here. Not going to implement inans or bool nans

hah!

that's what I meant :)

FYI also need to test case where u are replacing say True with a value (which prob will switch dtype)

weird just those no-op cases fail ... the other cases work

into the cave of replace we go :smile: 

@jreback Turns out it's not an issue with blocks or `replace` per se. In `com.mask_missing` string values to be masked are coerced to `bool`. If the replacement strings are non-empty then the mask is the original `bool` array. That's then passed to putmask eventually which of course replaces those `True` values with the string


Resolving patch:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -1216,6 +1216,7 @@ def should_store(self, value):
         return (issubclass(value.dtype.type, np.floating) and
                 value.dtype == self.dtype)
 
+
 class ComplexBlock(FloatOrComplexBlock):
     is_complex = True
 
@@ -1355,6 +1356,14 @@ def _try_cast(self, element):
     def should_store(self, value):
         return issubclass(value.dtype.type, np.bool_)
 
+    def replace(self, to_replace, value, inplace=False, filter=None,
+                regex=False):
+        to_replace_values = np.atleast_1d(to_replace)
+        if not np.can_cast(to_replace_values, bool):
+            to_replace = to_replace_values
+        return super(BoolBlock, self).replace(to_replace, value,
+                                              inplace=inplace, filter=filter,
+                                              regex=regex)
 
 class ObjectBlock(Block):
     is_object = True


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -7975,7 +7975,7 @@ def test_replace_dict_tuple_list_ordering_remains_the_same(self):
         tm.assert_frame_equal(res2, res3)
         tm.assert_frame_equal(res3, expected)
 
-    def test_replace_doesnt_replace_with_no_regex(self):
+    def test_replace_doesnt_replace_without_regex(self):
         from pandas.compat import StringIO
         raw = """fol T_opp T_Dir T_Enh
         0    1     0     0    vo
@@ -7986,6 +7986,29 @@ def test_replace_doesnt_replace_with_no_regex(self):
         res = df.replace({'\D': 1})
         tm.assert_frame_equal(df, res)
 
+    def test_replace_bool_with_string(self):
+        df = DataFrame({'a': [True, False], 'b': list('ab')})
+        result = df.replace(True, 'a')
+        expected = DataFrame({'a': ['a', False], 'b': df.b})
+        tm.assert_frame_equal(result, expected)
+
+    def test_replace_pure_bool_with_string_no_op(self):
+        df = DataFrame(np.random.rand(2, 2) > 0.5)
+        result = df.replace('asdf', 'fdsa')
+        tm.assert_frame_equal(df, result)
+
+    def test_replace_bool_with_bool(self):
+        df = DataFrame(np.random.rand(2, 2) > 0.5)
+        result = df.replace(False, True)
+        expected = DataFrame(np.ones((2, 2), dtype=bool))
+        tm.assert_frame_equal(result, expected)
+
+    def test_replace_with_dict_with_bool_keys(self):
+        df = DataFrame({0: [True, False], 1: [False, True]})
+        result = df.replace({'asdf': 'asdb', True: 'yes'})
+        expected = DataFrame({0: ['yes', False], 1: [False, 'yes']})
+        tm.assert_frame_equal(expected, result)
+
     def test_combine_multiple_frames_dtypes(self):
         from pandas import concat
 
diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py
--- a/pandas/tests/test_series.py
+++ b/pandas/tests/test_series.py
@@ -5291,7 +5291,6 @@ def test_replace(self):
         result = ser.replace(Timestamp('20130103'), Timestamp('20120101'))
         assert_series_equal(result, expected)
 
-
     def test_replace_with_single_list(self):
         ser = Series([0, 1, 2, 3, 4])
         result = ser.replace([1,2,3])
@@ -5307,6 +5306,7 @@ def test_replace_with_single_list(self):
             s.replace([1,2,3],inplace=True,method='crash_cymbal')
         assert_series_equal(s, ser)
 
+
     def test_replace_mixed_types(self):
         s = Series(np.arange(5),dtype='int64')
 
@@ -5349,6 +5349,30 @@ def check_replace(to_rep, val, expected):
         assert_series_equal(r, Series([1.0,2,'a'] +
                                       dr[3:].tolist(),dtype=object))
 
+    def test_replace_bool_with_string_no_op(self):
+        s = Series([True, False, True])
+        result = s.replace('fun', 'in-the-sun')
+        tm.assert_series_equal(s, result)
+
+    def test_replace_bool_with_string(self):
+        # nonexistent elements
+        s = Series([True, False, True])
+        result = s.replace(True, '2u')
+        expected = Series(['2u', False, '2u'])
+        tm.assert_series_equal(expected, result)
+
+    def test_replace_bool_with_bool(self):
+        s = Series([True, False, True])
+        result = s.replace(True, False)
+        expected = Series([False] * len(s))
+        tm.assert_series_equal(expected, result)
+
+    def test_replace_with_dict_with_bool_keys(self):
+        s = Series([True, False, True])
+        result = s.replace({'asdf': 'asdb', True: 'yes'})
+        expected = Series(['yes', False, 'yes'])
+        tm.assert_series_equal(expected, result)
+
     def test_asfreq(self):
         ts = Series([0., 1., 2.], index=[datetime(2009, 10, 30),
                                          datetime(2009, 11, 30),