Repository: pandas-dev/pandas

Issue:
Segfault when modifying pandas.DataFrame in-place after creating from numpy recarray
The following code generates a segfault when `use_records` is True.  This segfault only occurs when the DataFrame is generated from a record array and then I attempt to modify the series in-place.  This was not an issue in the previous release (v0.12) of Pandas.  Tested this code against 0.13.0-268-g08c1302.

```
import numpy as np
import pandas

data = [('right', 'left', 'left', 'left', 'right', 'left', 'timeout')]

use_records = True
if use_records:
    recarray = np.rec.fromarrays(data, names=['response'])
    df = pandas.DataFrame(recarray)
else:
    df = pandas.DataFrame({'response': data[0]})
mask = df.response == 'timeout'
df.response[mask] = 'none'
```



Discussion:
prob need to copy the rec array elements as they go in; numpy might be sharing it somehow; this doesn't happen with a regular numpy array, but maybe the sharing is different with a rec array.

want to do a PR for this?

I upgraded Numpy 1.7.1 to 1.8 and the problem went away (BTW, the problem was present even for regular arrays, so it must have been a bug in Numpy 1.7.1 that was fixed).

can you show what you were doing for a regular array? this shouldn't be the case....going to reopen (seg faults are bad...)

This will trigger the segfault as well (Numpy=1.7.1 & 1.7.2, Pandas=0.13.0):

```
import numpy as np
import pandas
data = ['right', 'left', 'left', 'left', 'right', 'left', 'timeout']
df = pandas.DataFrame({'response': np.array(data)})
mask = df.response == 'timeout'
df.response[mask] = 'none'
print df
```

Note that the `print df` is required to generate the segfault.

You are assigning over a view, see here: http://pandas.pydata.org/pandas-docs/dev/indexing.html#indexing-view-versus-copy

Do this instead (you NEVER want to do kind of a chained assignment!).

```
In [5]: df.loc[mask,'response'] = 'none'

In [6]: df
Out[6]: 
  response
0    right
1     left
2     left
3     left
4    right
5     left
6     none

[7 rows x 1 columns]
```

Its exceedingly difficult for pandas to even detect this. I will look at this, but generally this is a bad idea.


Resolving patch:
diff --git a/pandas/__init__.py b/pandas/__init__.py
--- a/pandas/__init__.py
+++ b/pandas/__init__.py
@@ -29,6 +29,7 @@
 _np_version = np.version.short_version
 _np_version_under1p6 = LooseVersion(_np_version) < '1.6'
 _np_version_under1p7 = LooseVersion(_np_version) < '1.7'
+_np_version_under1p8 = LooseVersion(_np_version) < '1.8'
 
 from pandas.version import version as __version__
 from pandas.info import __doc__
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -1005,9 +1005,14 @@ def _box_item_values(self, key, values):
         raise NotImplementedError
 
     def _maybe_cache_changed(self, item, value):
-        """ the object has called back to us saying
-        maybe it has changed """
-        self._data.set(item, value)
+        """
+        the object has called back to us saying
+        maybe it has changed
+
+        numpy < 1.8 has an issue with object arrays and aliasing
+        GH6026
+        """
+        self._data.set(item, value, check=pd._np_version_under1p8)
 
     @property
     def _is_cached(self):
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -26,7 +26,6 @@
 from pandas.compat import range, lrange, lmap, callable, map, zip, u
 from pandas.tseries.timedeltas import _coerce_scalar_to_timedelta_type
 
-
 class Block(PandasObject):
 
     """
@@ -279,7 +278,7 @@ def get(self, item):
     def iget(self, i):
         return self.values[i]
 
-    def set(self, item, value):
+    def set(self, item, value, check=False):
         """
         Modify Block in-place with new item value
 
@@ -1360,6 +1359,26 @@ def convert(self, convert_dates=True, convert_numeric=True, convert_timedeltas=T
 
         return blocks
 
+    def set(self, item, value, check=False):
+        """
+        Modify Block in-place with new item value
+
+        Returns
+        -------
+        None
+        """
+
+        loc = self.items.get_loc(item)
+
+        # GH6026
+        if check:
+            try:
+                if (self.values[loc] == value).all():
+                    return
+            except:
+                pass
+        self.values[loc] = value
+
     def _maybe_downcast(self, blocks, downcast=None):
 
         if downcast is not None:
@@ -1601,7 +1620,7 @@ def astype(self, dtype, copy=False, raise_on_error=True):
         return self._astype(dtype, copy=copy, raise_on_error=raise_on_error,
                             klass=klass)
 
-    def set(self, item, value):
+    def set(self, item, value, check=False):
         """
         Modify Block in-place with new item value
 
@@ -1714,7 +1733,7 @@ def prepare_for_merge(self, **kwargs):
     def post_merge(self, items, **kwargs):
         return self
 
-    def set(self, item, value):
+    def set(self, item, value, check=False):
         self.values = value
 
     def get(self, item):
@@ -2879,10 +2898,11 @@ def delete(self, item):
         if not is_unique:
             self._consolidate_inplace()
 
-    def set(self, item, value):
+    def set(self, item, value, check=False):
         """
         Set new item in-place. Does not consolidate. Adds new Block if not
         contained in the current set of items
+        if check, then validate that we are not setting the same data in-place
         """
         if not isinstance(value, SparseArray):
             if value.ndim == self.ndim - 1:
@@ -2898,7 +2918,7 @@ def _set_item(item, arr):
                 self._delete_from_block(i, item)
                 self._add_new_block(item, arr, loc=None)
             else:
-                block.set(item, arr)
+                block.set(item, arr, check=check)
 
         try:
 


Test patch:
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -1950,6 +1950,29 @@ def test_setitem_cache_updating(self):
             self.assert_(df.ix[0,'c'] == 0.0)
             self.assert_(df.ix[7,'c'] == 1.0)
 
+    def test_setitem_chained_setfault(self):
+
+        # GH6026
+        # setfaults under numpy 1.7.1 (ok on 1.8)
+        data = ['right', 'left', 'left', 'left', 'right', 'left', 'timeout']
+        mdata = ['right', 'left', 'left', 'left', 'right', 'left', 'none']
+
+        df = DataFrame({'response': np.array(data)})
+        mask = df.response == 'timeout'
+        df.response[mask] = 'none'
+        assert_frame_equal(df, DataFrame({'response': mdata }))
+
+        recarray = np.rec.fromarrays([data], names=['response'])
+        df = DataFrame(recarray)
+        mask = df.response == 'timeout'
+        df.response[mask] = 'none'
+        assert_frame_equal(df, DataFrame({'response': mdata }))
+
+        df = DataFrame({'response': data, 'response1' : data })
+        mask = df.response == 'timeout'
+        df.response[mask] = 'none'
+        assert_frame_equal(df, DataFrame({'response': mdata, 'response1' : data }))
+
     def test_detect_chained_assignment(self):
 
         pd.set_option('chained_assignment','raise')