Repository: pandas-dev/pandas

Issue:
ValueError on dropna call
I'm running into a problem with using `dropna` with a list of columns specified by `subset`. I wish to remove rows with missing values:

![missing](http://d.pr/i/5wgR+)

The dataset looks like this:

![dataset](http://d.pr/i/wRYE+)

However, when I try to call `dropna` using these columns, it complains about a duplicate axis:

```
variables.dropna(subset=['cigarette_smokers', 'birth_wt_child', 
                                 'oxygen', 'length_of_stay', 'hospitalized_vitamin_d', 'breastfed'])

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-168-96be0b0b00e8> in <module>()
      1 variables.dropna(subset=['cigarette_smokers', 'birth_wt_child', 
----> 2                                      'oxygen', 'length_of_stay', 'hospitalized_vitamin_d', 'breastfed'])

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in dropna(self, axis, how, thresh, subset, inplace)
   2407             if subset is not None:
   2408                 agg_axis_name = self._get_axis_name(agg_axis)
-> 2409                 agg_obj = self.reindex(**{agg_axis_name: subset})
   2410 
   2411             count = agg_obj.count(axis=agg_axis)

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in reindex(self, index, columns, **kwargs)
   2160     def reindex(self, index=None, columns=None, **kwargs):
   2161         return super(DataFrame, self).reindex(index=index, columns=columns,
-> 2162                                               **kwargs)
   2163 
   2164     @Appender(_shared_docs['reindex_axis'] % _shared_doc_kwargs)

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/generic.pyc in reindex(self, *args, **kwargs)
   1561         return self._reindex_axes(axes, level, limit,
   1562                                   method, fill_value, copy,
-> 1563                                   takeable=takeable).__finalize__(self)
   1564 
   1565     def _reindex_axes(self, axes, level, limit, method, fill_value, copy,

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in _reindex_axes(self, axes, level, limit, method, fill_value, copy, takeable)
   2110         if columns is not None:
   2111             frame = frame._reindex_columns(columns, copy, level, fill_value,
-> 2112                                            limit, takeable=takeable)
   2113 
   2114         index = axes['index']

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in _reindex_columns(self, new_columns, copy, level, fill_value, limit, takeable)
   2137         return self._reindex_with_indexers({1: [new_columns, indexer]},
   2138                                            copy=copy, fill_value=fill_value,
-> 2139                                            allow_dups=takeable)
   2140 
   2141     def _reindex_multi(self, axes, copy, fill_value):

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/generic.pyc in _reindex_with_indexers(self, reindexers, method, fill_value, limit, copy, allow_dups)
   1687                 new_data = new_data.reindex_indexer(index, indexer, axis=baxis,
   1688                                                     fill_value=fill_value,
-> 1689                                                     allow_dups=allow_dups)
   1690 
   1691             elif (baxis == 0 and index is not None and

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/internals.pyc in reindex_indexer(self, new_axis, indexer, axis, fill_value, allow_dups)
   3227         # trying to reindex on an axis with duplicates
   3228         if not allow_dups and not self.axes[axis].is_unique:
-> 3229             raise ValueError("cannot reindex from a duplicate axis")
   3230 
   3231         if not self.is_consolidated():

ValueError: cannot reindex from a duplicate axis
```

Running a current build from master in Python 2.7.5 on OS X 10.9.1.



Discussion:
can you provide the dataset?

[Here it is](http://d.pr/f/kaYU).

Seems to work on on 0.13.1 and master.

The numbers are different that yours though....?

```
In [4]: df = read_csv('variables.csv',index_col=0)

In [5]: df[['cigarette_smokers', 'birth_wt_child','oxygen', 'length_of_stay', 'hospitalized_vitamin_d', 'breastfed']].isnull().sum()
Out[5]: 
cigarette_smokers           1
birth_wt_child              2
oxygen                     31
length_of_stay             29
hospitalized_vitamin_d    480
breastfed                   0
dtype: int64

In [6]: df.dropna(subset=['cigarette_smokers', 'birth_wt_child','oxygen', 'length_of_stay', 'hospitalized_vitamin_d', 'breastfed']).info()
<class 'pandas.core.frame.DataFrame'>
Index: 2662 entries, A0001 to D3774
Data columns (total 19 columns):
daycare                   2661 non-null float64
hospitalized_vitamin_d    2662 non-null float64
gest_age                  2662 non-null int64
heart_hx                  2662 non-null int64
prev_cond                 2662 non-null int64
cigarette_smokers         2662 non-null float64
cigarette_preg            2662 non-null int64
sex_child                 2662 non-null int64
birth_wt_child            2662 non-null float64
breastfed                 2662 non-null int64
severity_score            2661 non-null float64
oxygen                    2662 non-null float64
days_oxygen               855 non-null float64
vent                      2661 non-null float64
days_vent                 106 non-null float64
icu_any                   2662 non-null bool
length_of_stay            2662 non-null float64
death                     2662 non-null bool
severity_score.1          2661 non-null float64
dtypes: bool(2), float64(11), int64(6)
```

Still does not work here on '0.13.0-395-gef55e60'. Will try to update.

maybe show me your exact code from reading on

So, this is odd: if I export the data frame to csv and reimport it, the `dropna()` call works fine. On the original data frame, it does not. If I look at the `info()` for the original data frame, I get the following `AssertionError`:

```
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-65-358eedd3f738> in <module>()
----> 1 variables.info()

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in info(self, verbose, buf, max_cols)
   1441             tmpl = "%s%s"
   1442             if show_counts:
-> 1443                 counts = self.count()
   1444                 if len(cols) != len(counts):  # pragma: no cover
   1445                     raise AssertionError('Columns must equal counts (%d != %d)' %

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/frame.pyc in count(self, axis, level, numeric_only)
   3857                 result = Series(counts, index=frame._get_agg_axis(axis))
   3858             else:
-> 3859                 result = notnull(frame).sum(axis=axis)
   3860 
   3861         return result.astype('int64')

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/common.pyc in notnull(obj)
    273         is given which of the element is *not* null.
    274     """
--> 275     res = isnull(obj)
    276     if np.isscalar(res):
    277         return not res

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/common.pyc in isnull(obj)
    125         given which of the element is null.
    126     """
--> 127     return _isnull(obj)
    128 
    129 

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/common.pyc in _isnull_new(obj)
    137         return _isnull_ndarraylike(obj)
    138     elif isinstance(obj, ABCGeneric):
--> 139         return obj._constructor(obj._data.apply(lambda x: isnull(x.values)))
    140     elif isinstance(obj, list) or hasattr(obj, '__array__'):
    141         return _isnull_ndarraylike(np.asarray(obj))

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/internals.pyc in apply(self, f, *args, **kwargs)
   2371             return self.make_empty(axes or self.axes)
   2372         bm = self.__class__(result_blocks, axes or self.axes,
-> 2373                             do_integrity_check=do_integrity_check)
   2374         bm._consolidate_inplace()
   2375         return bm

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/internals.pyc in __init__(self, blocks, axes, do_integrity_check, fastpath)
   2024         # we have a duplicate items index, setup the block maps
   2025         if not self.items.is_unique:
-> 2026             self._set_ref_locs(do_refs=True)
   2027 
   2028     def make_empty(self, axes=None):

/Library/Python/2.7/site-packages/pandas-0.13.0_395_gef55e60-py2.7-macosx-10.9-intel.egg/pandas/core/internals.pyc in _set_ref_locs(self, labels, do_refs)
   2164                             'Cannot create BlockManager._ref_locs because '
   2165                             'block [%s] with duplicate items [%s] does not '
-> 2166                             'have _ref_locs set' % (block, labels))
   2167 
   2168                     m = maybe_create_block_in_items_map(im, block)

AssertionError: Cannot create BlockManager._ref_locs because block [BoolBlock: [daycare, hospitalized_vitamin_d, cigarette_smokers, birth_wt_child, oxygen, days_oxygen, vent, days_vent, length_of_stay], 9 x 3169, dtype: bool] with duplicate items [Index([u'daycare', u'hospitalized_vitamin_d', u'gest_age', u'heart_hx', u'prev_cond', u'cigarette_smokers', u'cigarette_preg', u'sex_child', u'birth_wt_child', u'breastfed', u'severity_score', u'oxygen', u'days_oxygen', u'vent', u'days_vent', u'icu_any', u'length_of_stay', u'death', u'severity_score'], dtype='object')] does not have _ref_locs set
```

On the frame imported from the csv, it displays the info without error. Hope that provides some clues.

so...this indicates a failure in an operation before this (the printing causes the error to surface).

So do this. Start your process, then print out the frame (`str(df)`) every once in a while. when it raises, look tot he immediately prior operation. post that (or the whole sequence if you can).

Calling `str` on the data frame never raises an error. [Here is the whole notebook](http://nbviewer.ipython.org/gist/fonnesbeck/8928121) for reference.

can you show info on both of these frames

```
variables = pd.concat([covariates, severity], axis=1)
variables.shape
```

[I've updated the notebook](http://nbviewer.ipython.org/gist/fonnesbeck/8928121) with these calls. Thanks.

looks like `severity_score` is duplicated

Aha, yes. And importing via `read_csv` appends `.1` to the duplicate, so no exception is generated. The exception makes sense now, though its baffling when you don't know what you're looking for. I wonder if the message could be tweaked?


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -2414,8 +2414,8 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None,
 
             agg_obj = self
             if subset is not None:
-                agg_axis_name = self._get_axis_name(agg_axis)
-                agg_obj = self.reindex(**{agg_axis_name: subset})
+                ax = self._get_axis(agg_axis)
+                agg_obj = self.take(ax.get_indexer_for(subset),axis=agg_axis)
 
             count = agg_obj.count(axis=agg_axis)
 
diff --git a/pandas/core/index.py b/pandas/core/index.py
--- a/pandas/core/index.py
+++ b/pandas/core/index.py
@@ -1210,6 +1210,12 @@ def get_indexer_non_unique(self, target, **kwargs):
         indexer, missing = self._engine.get_indexer_non_unique(tgt_values)
         return Index(indexer), missing
 
+    def get_indexer_for(self, target, **kwargs):
+        """ guaranteed return of an indexer even when non-unique """
+        if self.is_unique:
+            return self.get_indexer(target, **kwargs)
+        return self.get_indexer_non_unique(target, **kwargs)[0]
+
     def _possibly_promote(self, other):
         # A hack, but it works
         from pandas.tseries.index import DatetimeIndex


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -3256,6 +3256,23 @@ def test_column_dups2(self):
         result = df2.drop('C',axis=1)
         assert_frame_equal(result, expected)
 
+        # dropna
+        df = DataFrame({'A' : np.random.randn(5),
+                        'B' : np.random.randn(5),
+                        'C' : np.random.randn(5),
+                        'D' : ['a','b','c','d','e'] })
+        df.iloc[2,[0,1,2]] = np.nan
+        df.iloc[0,0] = np.nan
+        df.iloc[1,1] = np.nan
+        df.iloc[:,3] = np.nan
+        expected = df.dropna(subset=['A','B','C'],how='all')
+        expected.columns = ['A','A','B','C']
+
+        df.columns = ['A','A','B','C']
+
+        result = df.dropna(subset=['A','C'],how='all')
+        assert_frame_equal(result, expected)
+
     def test_column_dups_indexing(self):
 
         def check(result, expected=None):