Repository: pandas-dev/pandas

Issue:
pandas concat working in 0.12 but not 0.13
I have a long list of dataframes that concat fine in 0.12 but not in 0.13.  Has anyone seen this error?
I am concatenating using the following

```
pd.concat(all_gics, ignore_index=True) 
```

My traceback:

/usr/anaconda/lib/python2.7/site-packages/pandas/tools/merge.pyc in concat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity)
    925                        keys=keys, levels=levels, names=names,
    926                        verify_integrity=verify_integrity)
--> 927     return op.get_result()
    928 
    929 

/usr/anaconda/lib/python2.7/site-packages/pandas/tools/merge.pyc in get_result(self)
   1016             return tmpdf
   1017         else:
-> 1018             new_data = self._get_concatenated_data()
   1019             new_data = self._post_merge(new_data)
   1020             return self.objs[0]._from_axes(new_data, self.new_axes)

/usr/anaconda/lib/python2.7/site-packages/pandas/tools/merge.pyc in _get_concatenated_data(self)
   1075             new_data = {}
   1076             for item in self.new_axes[0]:
-> 1077                 new_data[item] = self._concat_single_item(rdata, item)
   1078 
   1079         return new_data

/usr/anaconda/lib/python2.7/site-packages/pandas/tools/merge.pyc in _concat_single_item(self, objs, item)
   1184                     elif isnull(v).all():
   1185 
-> 1186                         alls.remove('other')
   1187                         alls.remove('object')
   1188 

KeyError: 'other'



Discussion:
pls put up a reproducible example

I will try to create an example that mimics it. (Its a private data source or else I would immediately post it immediately.)   


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -325,15 +325,16 @@ def _init_dict(self, data, index, columns, dtype=None):
     def _init_ndarray(self, values, index, columns, dtype=None,
                       copy=False):
         if isinstance(values, Series):
-            if columns is None and values.name is not None:
-                columns = [values.name]
+            if columns is None:
+                if values.name is not None:
+                    columns = [values.name]
             if index is None:
                 index = values.index
             else:
                 values = values.reindex(index)
 
             # zero len case (GH #2234)
-            if not len(values) and len(columns):
+            if not len(values) and columns is not None and len(columns):
                 values = np.empty((0, 1), dtype=object)
 
         values = _prep_ndarray(values, copy=copy)
diff --git a/pandas/tools/merge.py b/pandas/tools/merge.py
--- a/pandas/tools/merge.py
+++ b/pandas/tools/merge.py
@@ -1139,52 +1139,55 @@ def _concat_blocks(self, blocks):
 
     def _concat_single_item(self, objs, item):
         # this is called if we don't have consistent dtypes in a row-wise append
-
         all_values = []
-        dtypes = set()
+        dtypes = []
+        alls = set()
 
+        # figure out the resulting dtype of the combination
         for data, orig in zip(objs, self.objs):
+            d = dict([ (t,False) for t in ['object','datetime','timedelta','other'] ])
             if item in orig:
                 values = data.get(item)
                 if hasattr(values,'to_dense'):
                     values = values.to_dense()
-                dtypes.add(values.dtype)
                 all_values.append(values)
-            else:
-                all_values.append(None)
 
-        # figure out the resulting dtype of the combination
-        alls = set()
-        seen = []
-        for dtype in dtypes:
-            d = dict([ (t,False) for t in ['object','datetime','timedelta','other'] ])
-            if issubclass(dtype.type, (np.object_, np.bool_)):
-                d['object'] = True
-                alls.add('object')
-            elif is_datetime64_dtype(dtype):
-                d['datetime'] = True
-                alls.add('datetime')
-            elif is_timedelta64_dtype(dtype):
-                d['timedelta'] = True
-                alls.add('timedelta')
+                dtype = values.dtype
+
+                if issubclass(dtype.type, (np.object_, np.bool_)):
+                    d['object'] = True
+                    alls.add('object')
+                elif is_datetime64_dtype(dtype):
+                    d['datetime'] = True
+                    alls.add('datetime')
+                elif is_timedelta64_dtype(dtype):
+                    d['timedelta'] = True
+                    alls.add('timedelta')
+                else:
+                    d['other'] = True
+                    alls.add('other')
+
             else:
+                all_values.append(None)
                 d['other'] = True
                 alls.add('other')
-            seen.append(d)
+
+            dtypes.append(d)
 
         if 'datetime' in alls or 'timedelta' in alls:
 
             if 'object' in alls or 'other' in alls:
-                for v, s in zip(all_values,seen):
-                    if s.get('datetime') or s.get('timedelta'):
+
+                for v, d in zip(all_values,dtypes):
+                    if d.get('datetime') or d.get('timedelta'):
                         pass
 
                     # if we have all null, then leave a date/time like type
                     # if we have only that type left
-                    elif isnull(v).all():
+                    elif v is None or isnull(v).all():
 
-                        alls.remove('other')
-                        alls.remove('object')
+                        alls.discard('other')
+                        alls.discard('object')
 
         # create the result
         if 'object' in alls:
@@ -1200,7 +1203,7 @@ def _concat_single_item(self, objs, item):
 
         to_concat = []
         for obj, item_values in zip(objs, all_values):
-            if item_values is None:
+            if item_values is None or isnull(item_values).all():
                 shape = obj.shape[1:]
                 missing_arr = np.empty(shape, dtype=empty_dtype)
                 missing_arr.fill(fill_value)


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -6154,6 +6154,48 @@ def test_append_empty_dataframe(self):
         expected = df1.copy()
         assert_frame_equal(result, expected)
 
+    def test_append_dtypes(self):
+
+        # GH 5754
+        # row appends of different dtypes (so need to do by-item)
+        # can sometimes infer the correct type
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(5))
+        df2 = DataFrame()
+        result = df1.append(df2)
+        expected = df1.copy()
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : 'foo' }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : [ Timestamp('20130101'), 'foo' ]})
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : np.nan }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), np.nan ],dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : np.nan }, index=lrange(1,2), dtype=object)
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), np.nan ],dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : np.nan }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ np.nan, Timestamp('20130101')] ,dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : 1 }, index=lrange(1,2), dtype=object)
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), 1 ]) })
+        assert_frame_equal(result, expected)
+
     def test_asfreq(self):
         offset_monthly = self.tsframe.asfreq(datetools.bmonthEnd)
         rule_monthly = self.tsframe.asfreq('BM')
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -1793,6 +1793,14 @@ def f():
         expected = DataFrame(columns=['A','B','C'])
         assert_frame_equal(result,expected)
 
+        # GH 5756
+        # setting with empty Series
+        df = DataFrame(Series())
+        assert_frame_equal(df, DataFrame({ 0 : Series() }))
+
+        df = DataFrame(Series(name='foo'))
+        assert_frame_equal(df, DataFrame({ 'foo' : Series() }))
+
     def test_cache_updating(self):
         # GH 4939, make sure to update the cache on setitem