Repository: pandas-dev/pandas

Issue:
API: New callable indexing makes storing functions in a Series difficult
Example of something that previously worked (before 7bbd031104ee161b2fb79ba6f5732910661f94f8):

``` .python
>>> s = pd.Series([lambda x: x] * 10)
>>> s[s.index > 5] = lambda x: x + 1
```

But now the second line tries to apply the function in the rhs to the elements of the Series, rather than assigning them (and throws an exception). This is very counter-intuitive when using `__setitem__` rather than calling `Series.where` directly.



Discussion:
this is quite odd to store callables in a Series

what is the purpose of doing that?

hmm, this is simple to fix (we just don't evaluate a callable for `other` in `.where`).

@sinhrks do we have a valid use of `other` as a callable?

We're choosing a function based on some columns in a DataFrame, and then applying them on other column values / external data. Since there might be a different function for each row, it's convenient to store them in the same place.

@evanpw I think you might be able to just remove the `_maybe_callable(other...)` and this will work. Not really sure of the original purpose of testing for a callable on the other (e.g. it works, but IMHO not that useful).

If you want to give this a shot and see what it looks like would be great.

Yes, deleting this line from `where` in generic.py fixes the problem for me:

``` .python
other = com._apply_if_callable(other, self)
```

The usage in my mind was chaining like:

```
pd.DataFrame({'A': [1, 2, 1, 2], 'B': [1, 2, 3, 4]}).groupby('A').sum().where(lambda x: x < 5, lambda x: -x)
#    B
# A   
# 1  4
# 2 -6
```

I think it is more likely rather than storing callable.

CC: @TomAugspurger

I think that makes sense when calling `where` directly. What about making it configurable, and turned off when using `__setitem__`?

yes that is a bug I think.


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -2395,7 +2395,7 @@ def _setitem_frame(self, key, value):
 
         self._check_inplace_setting(value)
         self._check_setitem_copy()
-        self.where(-key, value, inplace=True)
+        self._where(-key, value, inplace=True)
 
     def _ensure_valid_index(self, value):
         """
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -4419,54 +4419,14 @@ def _align_series(self, other, join='outer', axis=None, level=None,
             right = right.fillna(fill_value, method=method, limit=limit)
         return left.__finalize__(self), right.__finalize__(other)
 
-    _shared_docs['where'] = ("""
-        Return an object of same shape as self and whose corresponding
-        entries are from self where cond is %(cond)s and otherwise are from
-        other.
-
-        Parameters
-        ----------
-        cond : boolean %(klass)s, array or callable
-            If cond is callable, it is computed on the %(klass)s and
-            should return boolean %(klass)s or array.
-            The callable must not change input %(klass)s
-            (though pandas doesn't check it).
-
-            .. versionadded:: 0.18.1
-
-            A callable can be used as cond.
-
-        other : scalar, %(klass)s, or callable
-            If other is callable, it is computed on the %(klass)s and
-            should return scalar or %(klass)s.
-            The callable must not change input %(klass)s
-            (though pandas doesn't check it).
-
-            .. versionadded:: 0.18.1
-
-            A callable can be used as other.
-
-        inplace : boolean, default False
-            Whether to perform the operation in place on the data
-        axis : alignment axis if needed, default None
-        level : alignment level if needed, default None
-        try_cast : boolean, default False
-            try to cast the result back to the input type (if possible),
-        raise_on_error : boolean, default True
-            Whether to raise on invalid data types (e.g. trying to where on
-            strings)
-
-        Returns
-        -------
-        wh : same type as caller
-        """)
-
-    @Appender(_shared_docs['where'] % dict(_shared_doc_kwargs, cond="True"))
-    def where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
-              try_cast=False, raise_on_error=True):
+    def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
+               try_cast=False, raise_on_error=True):
+        """
+        Equivalent to public method `where`, except that `other` is not
+        applied as a function even if callable. Used in __setitem__.
+        """
 
         cond = com._apply_if_callable(cond, self)
-        other = com._apply_if_callable(other, self)
 
         if isinstance(cond, NDFrame):
             cond, _ = cond.align(self, join='right', broadcast_axis=1)
@@ -4622,6 +4582,56 @@ def where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
 
             return self._constructor(new_data).__finalize__(self)
 
+    _shared_docs['where'] = ("""
+        Return an object of same shape as self and whose corresponding
+        entries are from self where cond is %(cond)s and otherwise are from
+        other.
+
+        Parameters
+        ----------
+        cond : boolean %(klass)s, array or callable
+            If cond is callable, it is computed on the %(klass)s and
+            should return boolean %(klass)s or array.
+            The callable must not change input %(klass)s
+            (though pandas doesn't check it).
+
+            .. versionadded:: 0.18.1
+
+            A callable can be used as cond.
+
+        other : scalar, %(klass)s, or callable
+            If other is callable, it is computed on the %(klass)s and
+            should return scalar or %(klass)s.
+            The callable must not change input %(klass)s
+            (though pandas doesn't check it).
+
+            .. versionadded:: 0.18.1
+
+            A callable can be used as other.
+
+        inplace : boolean, default False
+            Whether to perform the operation in place on the data
+        axis : alignment axis if needed, default None
+        level : alignment level if needed, default None
+        try_cast : boolean, default False
+            try to cast the result back to the input type (if possible),
+        raise_on_error : boolean, default True
+            Whether to raise on invalid data types (e.g. trying to where on
+            strings)
+
+        Returns
+        -------
+        wh : same type as caller
+        """)
+
+    @Appender(_shared_docs['where'] % dict(_shared_doc_kwargs, cond="True"))
+    def where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
+              try_cast=False, raise_on_error=True):
+
+        other = com._apply_if_callable(other, self)
+        return self._where(cond, other, inplace, axis, level, try_cast,
+                           raise_on_error)
+
     @Appender(_shared_docs['where'] % dict(_shared_doc_kwargs, cond="False"))
     def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
              try_cast=False, raise_on_error=True):
diff --git a/pandas/core/series.py b/pandas/core/series.py
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -738,7 +738,7 @@ def setitem(key, value):
             if is_bool_indexer(key):
                 key = check_bool_indexer(self.index, key)
                 try:
-                    self.where(~key, value, inplace=True)
+                    self._where(~key, value, inplace=True)
                     return
                 except InvalidIndexError:
                     pass


Test patch:
diff --git a/pandas/tests/frame/test_indexing.py b/pandas/tests/frame/test_indexing.py
--- a/pandas/tests/frame/test_indexing.py
+++ b/pandas/tests/frame/test_indexing.py
@@ -207,6 +207,16 @@ def test_setitem_callable(self):
         exp = pd.DataFrame({'A': [11, 12, 13, 14], 'B': [5, 6, 7, 8]})
         tm.assert_frame_equal(df, exp)
 
+    def test_setitem_other_callable(self):
+        # GH 13299
+        inc = lambda x: x + 1
+
+        df = pd.DataFrame([[-1, 1], [1, -1]])
+        df[df > 0] = inc
+
+        expected = pd.DataFrame([[-1, inc], [inc, -1]])
+        tm.assert_frame_equal(df, expected)
+
     def test_getitem_boolean(self):
         # boolean indexing
         d = self.tsframe.index[10]
diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py
--- a/pandas/tests/series/test_indexing.py
+++ b/pandas/tests/series/test_indexing.py
@@ -441,6 +441,16 @@ def test_setitem_callable(self):
         s[lambda x: 'A'] = -1
         tm.assert_series_equal(s, pd.Series([-1, 2, 3, 4], index=list('ABCD')))
 
+    def test_setitem_other_callable(self):
+        # GH 13299
+        inc = lambda x: x + 1
+
+        s = pd.Series([1, 2, -1, 4])
+        s[s < 0] = inc
+
+        expected = pd.Series([1, 2, inc, 4])
+        tm.assert_series_equal(s, expected)
+
     def test_slice(self):
         numSlice = self.series[10:20]
         numSliceEnd = self.series[-10:]