Repository: pandas-dev/pandas

Issue:
Trouble with NaNs: set_index().reset_index() corrupts data
In the following code, my data are corrupted by the sequence set_index().reset_index(). The value of "QC" is actually changed to 1 from NaN where it should be NaN.

Btw, for symmetry I added the ".reset_index()", but the data corruption is introduced by set_index.

This problem exists in both version '0.10.1' and '0.11.0'

```
    bug=pd.DataFrame({'PRuid': {17: 'nonQC', 18: 'nonQC', 19: 'nonQC', 20: '10', 21: '11', 22: '12', 23: '13', 24: '24', 25: '35', 26: '46', 27: '47', 28: '48', 29: '59', 30: '10'}, 'QC': {17: 0.0, 18: 0.0, 19: 0.0, 20: nan, 21: nan, 22: nan, 23: nan, 24: 1.0, 25: nan, 26: nan, 27: nan, 28: nan, 29: nan, 30: nan}, 'data': {17: 7.9544899999999998, 18: 8.0142609999999994, 19: 7.8591520000000008, 20: 0.86140349999999999, 21: 0.87853110000000001, 22: 0.8427041999999999, 23: 0.78587700000000005, 24: 0.73062459999999996, 25: 0.81668560000000001, 26: 0.81927080000000008, 27: 0.80705009999999999, 28: 0.81440240000000008, 29: 0.80140849999999997, 30: 0.81307740000000006}, 'year': {17: 2006, 18: 2007, 19: 2008, 20: 1985, 21: 1985, 22: 1985, 23: 1985, 24: 1985, 25: 1985, 26: 1985, 27: 1985, 28: 1985, 29: 1985, 30: 1986}})
    print bug
    #print bug.set_index(['year','PRuid','QC'])
    print bug.set_index(['year','PRuid','QC']).reset_index()
```



Discussion:
@cpbl What version are you using? I cannot reproduce this with git master.


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -2808,9 +2808,18 @@ def reset_index(self, level=None, drop=False, inplace=False, col_level=0,
         else:
             new_obj = self.copy()
 
-        def _maybe_cast(values):
+        def _maybe_cast(values, labels=None):
+
             if values.dtype == np.object_:
                 values = lib.maybe_convert_objects(values)
+
+            # if we have the labels, extract the values with a mask
+            if labels is not None:
+                mask = labels == -1
+                values = values.take(labels)
+                if mask.any():
+                    values, changed = com._maybe_upcast_putmask(values,mask,np.nan)
+
             return values
 
         new_index = np.arange(len(new_obj))
@@ -2843,9 +2852,9 @@ def _maybe_cast(values):
                             col_name = tuple(name_lst)
 
                     # to ndarray and maybe infer different dtype
-                    level_values = _maybe_cast(lev.values)
+                    level_values = _maybe_cast(lev.values, lab)
                     if level is None or i in level:
-                        new_obj.insert(0, col_name, level_values.take(lab))
+                        new_obj.insert(0, col_name, level_values)
 
         elif not drop:
             name = self.index.name
@@ -2865,8 +2874,8 @@ def _maybe_cast(values):
                   self.index.tz is not None):
                 values = self.index.asobject
             else:
-                values = self.index.values
-            new_obj.insert(0, name, _maybe_cast(values))
+                values = _maybe_cast(self.index.values)
+            new_obj.insert(0, name, values)
 
         new_obj.index = new_index
         if not inplace:


Test patch:
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -799,6 +799,25 @@ def test_indexing_mixed_frame_bug(self):
         self.assert_(df.iloc[0,2] == '-----')
 
         #if I look at df, then element [0,2] equals '_'. If instead I type df.ix[idx,'test'], I get '-----', finally by typing df.iloc[0,2] I get '_'.
+                
+
+    def test_set_index_nan(self):
+
+        # GH 3586
+        df = DataFrame({'PRuid': {17: 'nonQC', 18: 'nonQC', 19: 'nonQC', 20: '10', 21: '11', 22: '12', 23: '13', 
+                                  24: '24', 25: '35', 26: '46', 27: '47', 28: '48', 29: '59', 30: '10'}, 
+                        'QC': {17: 0.0, 18: 0.0, 19: 0.0, 20: nan, 21: nan, 22: nan, 23: nan, 24: 1.0, 25: nan, 
+                               26: nan, 27: nan, 28: nan, 29: nan, 30: nan}, 
+                        'data': {17: 7.9544899999999998, 18: 8.0142609999999994, 19: 7.8591520000000008, 20: 0.86140349999999999, 
+                                 21: 0.87853110000000001, 22: 0.8427041999999999, 23: 0.78587700000000005, 24: 0.73062459999999996,
+                                 25: 0.81668560000000001, 26: 0.81927080000000008, 27: 0.80705009999999999, 28: 0.81440240000000008, 
+                                 29: 0.80140849999999997, 30: 0.81307740000000006}, 
+                        'year': {17: 2006, 18: 2007, 19: 2008, 20: 1985, 21: 1985, 22: 1985, 23: 1985, 
+                                 24: 1985, 25: 1985, 26: 1985, 27: 1985, 28: 1985, 29: 1985, 30: 1986}}).reset_index()
+
+        result = df.set_index(['year','PRuid','QC']).reset_index().reindex(columns=df.columns)
+        assert_frame_equal(result,df)
+
 
 if __name__ == '__main__':
     import nose