Repository: pandas-dev/pandas

Issue:
BUG: concat of ExtensionArray with other dtype fails
```
In [22]: from pandas.tests.extension.decimal.array import DecimalArray, make_data
    ...: dec_arr = DecimalArray(make_data())
    ...: df1 = pd.DataFrame({'ext': dec_arr[:3]})

In [23]: df2 = pd.DataFrame({'ext': [1.0, 2.0]})

In [24]: pd.concat([df1, df2], axis=0)
---------------------------------------------------------------------------
AxisError                                 Traceback (most recent call last)
<ipython-input-24-e046f4a55b52> in <module>()
----> 1 pd.concat([df1, df2], axis=0)

/home/joris/scipy/pandas/pandas/core/reshape/concat.py in concat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, copy)
    211                        verify_integrity=verify_integrity,
    212                        copy=copy)
--> 213     return op.get_result()
    214 
    215 

/home/joris/scipy/pandas/pandas/core/reshape/concat.py in get_result(self)
    406             new_data = concatenate_block_managers(
    407                 mgrs_indexers, self.new_axes, concat_axis=self.axis,
--> 408                 copy=self.copy)
    409             if not self.copy:
    410                 new_data._consolidate_inplace()

/home/joris/scipy/pandas/pandas/core/internals.py in concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy)
   5415             print("--- Using concatenate_join_units")
   5416             b = make_block(
-> 5417                 concatenate_join_units(join_units, concat_axis, copy=copy),
   5418                 placement=placement)
   5419         blocks.append(b)

/home/joris/scipy/pandas/pandas/core/internals.py in concatenate_join_units(join_units, concat_axis, copy)
   5557             concat_values = concat_values.copy()
   5558     else:
-> 5559         concat_values = _concat._concat_compat(to_concat, axis=concat_axis)
   5560 
   5561     return concat_values

/home/joris/scipy/pandas/pandas/core/dtypes/concat.py in _concat_compat(to_concat, axis)
    189                 to_concat = [x.astype('object') for x in to_concat]
    190 
--> 191     return np.concatenate(to_concat, axis=axis)
    192 
    193 

AxisError: axis 1 is out of bounds for array of dimension 1
```

This is because `_concat_compat` is not yet aware of extension arrays, and hence does not reshape the extension array object values to 2D (as is done correctly in case of other non-consolidatable blocks like datetimetz)


Discussion:
I'm looking into this now. What's the expected behavior? I'd favor erroring with an informative message (rather than trying to cast the EA to object, or the other to an EA).
If we don't error, the obvious (to me) way to go is convert to object dtype and then concatenate. This is actually already happening (see the `to_concat = [x.astype('object') for x in to_concat]` in the error traceback at the bottom. The only thing that is missing there is a proper reshape to take into account that an ExtensionArray is stored as 1D instead of 2D like any other consolidatable block)
But I would also be fine with raising an informative error message. Given how we in pandas most of the time favor coercing to other (object) dtype instead of erroring, not raising an error seems the most consistent (but maybe we do this coercing a bit too much :-))
Yes, I think I'd like to be conservative here and not coerce. If we decide to coerce later (like we did for categorical IIRC) that's less disruptive than taking away coercion.
On the other hand, it's strange that pandas' EAs would convert while 3rd party ones wouldn't... Probably best to be consistent here.

Resolving patch:
diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py
--- a/pandas/core/dtypes/concat.py
+++ b/pandas/core/dtypes/concat.py
@@ -8,6 +8,7 @@
 from pandas.core.dtypes.common import (
     is_categorical_dtype,
     is_sparse,
+    is_extension_array_dtype,
     is_datetimetz,
     is_datetime64_dtype,
     is_timedelta64_dtype,
@@ -173,6 +174,10 @@ def is_nonempty(x):
     elif 'sparse' in typs:
         return _concat_sparse(to_concat, axis=axis, typs=typs)
 
+    extensions = [is_extension_array_dtype(x) for x in to_concat]
+    if any(extensions):
+        to_concat = [np.atleast_2d(x.astype('object')) for x in to_concat]
+
     if not nonempty:
         # we have all empties, but may need to coerce the result dtype to
         # object if we have non-numeric type operands (numpy would otherwise
@@ -210,7 +215,7 @@ def _concat_categorical(to_concat, axis=0):
 
     def _concat_asobject(to_concat):
         to_concat = [x.get_values() if is_categorical_dtype(x.dtype)
-                     else x.ravel() for x in to_concat]
+                     else np.asarray(x).ravel() for x in to_concat]
         res = _concat_compat(to_concat)
         if axis == 1:
             return res.reshape(1, len(res))
@@ -548,6 +553,8 @@ def convert_sparse(x, axis):
         # coerce to native type
         if isinstance(x, SparseArray):
             x = x.get_values()
+        else:
+            x = np.asarray(x)
         x = x.ravel()
         if axis > 0:
             x = np.atleast_2d(x)


Test patch:
diff --git a/pandas/tests/extension/base/reshaping.py b/pandas/tests/extension/base/reshaping.py
--- a/pandas/tests/extension/base/reshaping.py
+++ b/pandas/tests/extension/base/reshaping.py
@@ -41,6 +41,29 @@ def test_concat_all_na_block(self, data_missing, in_frame):
             expected = pd.Series(data_missing.take([1, 1, 0, 0]))
             self.assert_series_equal(result, expected)
 
+    def test_concat_mixed_dtypes(self, data):
+        # https://github.com/pandas-dev/pandas/issues/20762
+        df1 = pd.DataFrame({'A': data[:3]})
+        df2 = pd.DataFrame({"A": [1, 2, 3]})
+        df3 = pd.DataFrame({"A": ['a', 'b', 'c']}).astype('category')
+        df4 = pd.DataFrame({"A": pd.SparseArray([1, 2, 3])})
+        dfs = [df1, df2, df3, df4]
+
+        # dataframes
+        result = pd.concat(dfs)
+        expected = pd.concat([x.astype(object) for x in dfs])
+        self.assert_frame_equal(result, expected)
+
+        # series
+        result = pd.concat([x['A'] for x in dfs])
+        expected = pd.concat([x['A'].astype(object) for x in dfs])
+        self.assert_series_equal(result, expected)
+
+        # simple test for just EA and one other
+        result = pd.concat([df1, df2])
+        expected = pd.concat([df1.astype('object'), df2.astype('object')])
+        self.assert_frame_equal(result, expected)
+
     def test_align(self, data, na_value):
         a = data[:3]
         b = data[2:5]