Repository: pandas-dev/pandas

Issue:
Merge fails when dataframe contains categoricals
Trying to perform a left merge between two dataframes using a column of type object. If I include categoricals in the right dataframe, I get the following error. Trying to reproduce with a toy dataset but no luck so far.

``` python
out = pd.merge(left, right, how='left', left_on='left_id', right_on='right_id')
Traceback (most recent call last):
  File ".../pandas/tools/merge.py", line 39, in merge return op.get_result()
  File ".../pandas/tools/merge.py", line 201, in get_result concat_axis=0, copy=self.copy)
  File ".../pandas/core/internals.py", line 4046, in concatenate_block_managers for placement, join_units in concat_plan]
  File ".../pandas/core/internals.py", line 4135, in concatenate_join_units empty_dtype, upcasted_na = get_empty_dtype_and_na(join_units)
  File ".../pandas/core/internals.py", line 4074, in get_empty_dtype_and_na dtypes[i] = unit.dtype
  File ".../pandas/src/properties.pyx", line 34, in pandas.lib.cache_readonly.__get__ (pandas/lib.c:40664)
  File ".../pandas/core/internals.py", line 4349, in dtype self.block.fill_value)[0])
  File ".../pandas/core/common.py", line 1128, in _maybe_promote if issubclass(np.dtype(dtype).type, compat.string_types):
TypeError: data type not understood
```



Discussion:
pd.show_versions()

df.info() and df.head() for each frame

``` python
df = pd.merge(left, right, how='left', left_on='b', right_on='c')

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.9.final.0
python-bits: 64
OS: Linux
OS-release: 3.13.0-45-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: 0.15.2
nose: 1.3.4
Cython: 0.21
numpy: 1.9.1
scipy: 0.15.1
statsmodels: None
IPython: 2.3.1
sphinx: 1.2.3
patsy: 0.3.0
dateutil: 1.5
pytz: 2014.9
bottleneck: None
tables: 3.1.1
numexpr: 2.3.1
matplotlib: 1.4.2
openpyxl: 1.8.5
xlrd: 0.9.3
xlwt: 0.7.5
xlsxwriter: 0.5.7
lxml: 3.4.0
bs4: 4.3.2
html5lib: None
httplib2: None
apiclient: None
rpy2: 2.4.4
sqlalchemy: 0.9.7
pymysql: None
psycopg2: None
None
print left.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 29040 entries, 0 to 29039
Data columns (total 2 columns):
a    29040 non-null object
b    29040 non-null object
dtypes: object(2)
memory usage: 680.6+ KB
None
print left.head()
                    a                   b
0  00640000008PbqmAAC  0013000000CBGKbAAP
1  00640000008PbqmAAC  0013000000CBGKbAAP
2  00640000008PbqmAAC  0013000000CBGKbAAP
3  00640000008PbqmAAC  0013000000CBGKbAAP
4  00640000008PbqmAAC  0013000000CBGKbAAP
print right.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 2952 entries, 0 to 2951
Data columns (total 2 columns):
c    2952 non-null object
d    2952 non-null category
dtypes: category(1), object(1)
memory usage: 49.2+ KB
None
print right.head()
                    c     d
0  0014000000G3eszAAB  null
1  0014000000G3TTVAA3  null
2  0014000000G4H6yAAF  null
3  0014000000G4HpmAAF  null
4  0014000000G4IR8AAN  null
```

and you merging in the categorical column? iirc I think we allow this kind of object/cat merging (as the merge column) but would need a specifc example to see what the issue is

I'm merging on an object column and merging in a category column.
I have a reproducible example now:

``` python
right = pd.DataFrame({'c': {0: 'a', 1: 'b', 2: 'c', 3: 'd', 4: 'e'},
 'd': {0: 'null', 1: 'null', 2: 'null', 3: 'null', 4: 'null'}})
right['d'] = right['d'].astype('category')

left = pd.DataFrame({'a': {0: 'f', 1: 'f', 2: 'f', 3: 'f', 4: 'f'},
 'b': {0: 'g', 1: 'g', 2: 'g', 3: 'g', 4: 'g'}})
df = pd.merge(left, right, how='left', left_on='b', right_on='c')
```

hmm I don't think this is tested (only with concat). ok, marking as a bug. I think pretty easy to resolve though. You are welcome to dig in if you'd like.

I just ran into this in production code. Any hints on how this could be fixed? I'd gladly try.

FYI, I don't get this bug in 0.15.1

@lminer: Confirmed here, downgrading helped.

@lminer, thanks! Confirmed as well, working fine with 0.15.1.


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -1146,7 +1146,9 @@ def _maybe_promote(dtype, fill_value=np.nan):
         dtype = np.object_
 
     # in case we have a string that looked like a number
-    if issubclass(np.dtype(dtype).type, compat.string_types):
+    if is_categorical_dtype(dtype):
+        dtype = dtype
+    elif issubclass(np.dtype(dtype).type, compat.string_types):
         dtype = np.object_
 
     return dtype, fill_value
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -4327,8 +4327,9 @@ def dtype(self):
         if not self.needs_filling:
             return self.block.dtype
         else:
-            return np.dtype(com._maybe_promote(self.block.dtype,
-                                               self.block.fill_value)[0])
+            return com._get_dtype(com._maybe_promote(self.block.dtype,
+                                                     self.block.fill_value)[0])
+
         return self._dtype
 
     @cache_readonly


Test patch:
diff --git a/pandas/io/tests/test_data.py b/pandas/io/tests/test_data.py
--- a/pandas/io/tests/test_data.py
+++ b/pandas/io/tests/test_data.py
@@ -112,7 +112,7 @@ def test_get_multi2(self):
 
                 # sanity checking
 
-                assert np.issubdtype(result.dtype, np.floating)
+                self.assertTrue(np.issubdtype(result.dtype, np.floating))
                 result = pan.Open.ix['Jan-15-12':'Jan-20-12']
                 self.assertEqual((4, 3), result.shape)
                 assert_n_failed_equals_n_null_columns(w, result)
@@ -121,11 +121,11 @@ def test_get_multi2(self):
     def test_dtypes(self):
         #GH3995, #GH8980
         data = web.get_data_google('F', start='JAN-01-10', end='JAN-27-13')
-        assert np.issubdtype(data.Open.dtype, np.number)
-        assert np.issubdtype(data.Close.dtype, np.number)
-        assert np.issubdtype(data.Low.dtype, np.number)
-        assert np.issubdtype(data.High.dtype, np.number)
-        assert np.issubdtype(data.Volume.dtype, np.number)
+        self.assertTrue(np.issubdtype(data.Open.dtype, np.number))
+        self.assertTrue(np.issubdtype(data.Close.dtype, np.number))
+        self.assertTrue(np.issubdtype(data.Low.dtype, np.number))
+        self.assertTrue(np.issubdtype(data.High.dtype, np.number))
+        self.assertTrue(np.issubdtype(data.Volume.dtype, np.number))
 
     @network
     def test_unicode_date(self):
@@ -183,7 +183,7 @@ def test_get_components_dow_jones(self):
         raise nose.SkipTest('unreliable test, receive partial components back for dow_jones')
 
         df = web.get_components_yahoo('^DJI') #Dow Jones
-        assert isinstance(df, pd.DataFrame)
+        self.assertIsInstance(df, pd.DataFrame)
         self.assertEqual(len(df), 30)
 
     @network
@@ -191,7 +191,7 @@ def test_get_components_dax(self):
         raise nose.SkipTest('unreliable test, receive partial components back for dax')
 
         df = web.get_components_yahoo('^GDAXI') #DAX
-        assert isinstance(df, pd.DataFrame)
+        self.assertIsInstance(df, pd.DataFrame)
         self.assertEqual(len(df), 30)
         self.assertEqual(df[df.name.str.contains('adidas', case=False)].index,
                          'ADS.DE')
@@ -202,13 +202,13 @@ def test_get_components_nasdaq_100(self):
         raise nose.SkipTest('unreliable test, receive partial components back for nasdaq_100')
 
         df = web.get_components_yahoo('^NDX') #NASDAQ-100
-        assert isinstance(df, pd.DataFrame)
+        self.assertIsInstance(df, pd.DataFrame)
 
         if len(df) > 1:
             # Usual culprits, should be around for a while
-            assert 'AAPL' in df.index
-            assert 'GOOG' in df.index
-            assert 'AMZN' in df.index
+            self.assertTrue('AAPL' in df.index)
+            self.assertTrue('GOOG' in df.index)
+            self.assertTrue('AMZN' in df.index)
         else:
             expected = DataFrame({'exchange': 'N/A', 'name': '@^NDX'},
                                  index=['@^NDX'])
@@ -256,7 +256,7 @@ def test_get_data_multiple_symbols_two_dates(self):
         self.assertEqual(len(result), 3)
 
         # sanity checking
-        assert np.issubdtype(result.dtype, np.floating)
+        self.assertTrue(np.issubdtype(result.dtype, np.floating))
 
         expected = np.array([[18.99,  28.4, 25.18],
                              [18.58, 28.31, 25.13],
@@ -276,7 +276,7 @@ def test_get_date_ret_index(self):
             self.assertEqual(result, 1.0)
 
         # sanity checking
-        assert np.issubdtype(pan.values.dtype, np.floating)
+        self.assertTrue(np.issubdtype(pan.values.dtype, np.floating))
 
 
 class TestYahooOptions(tm.TestCase):
@@ -383,26 +383,26 @@ def test_get_underlying_price(self):
             quote_price = options_object._underlying_price_from_root(root)
         except RemoteDataError as e:
             raise nose.SkipTest(e)
-        self.assert_(isinstance(quote_price, float))
+        self.assertIsInstance(quote_price, float)
 
     def test_sample_page_price_quote_time1(self):
         #Tests the weekend quote time format
         price, quote_time = self.aapl._underlying_price_and_time_from_url(self.html1)
-        self.assert_(isinstance(price, (int, float, complex)))
-        self.assert_(isinstance(quote_time, (datetime, Timestamp)))
+        self.assertIsInstance(price, (int, float, complex))
+        self.assertIsInstance(quote_time, (datetime, Timestamp))
 
     def test_chop(self):
         #regression test for #7625
         self.aapl.chop_data(self.data1, above_below=2, underlying_price=np.nan)
         chopped = self.aapl.chop_data(self.data1, above_below=2, underlying_price=100)
-        self.assert_(isinstance(chopped, DataFrame))
+        self.assertIsInstance(chopped, DataFrame)
         self.assertTrue(len(chopped) > 1)
 
     def test_chop_out_of_strike_range(self):
         #regression test for #7625
         self.aapl.chop_data(self.data1, above_below=2, underlying_price=np.nan)
         chopped = self.aapl.chop_data(self.data1, above_below=2, underlying_price=100000)
-        self.assert_(isinstance(chopped, DataFrame))
+        self.assertIsInstance(chopped, DataFrame)
         self.assertTrue(len(chopped) > 1)
 
 
@@ -411,8 +411,8 @@ def test_sample_page_price_quote_time2(self):
         #Tests the EDT page format
         #regression test for #8741
         price, quote_time = self.aapl._underlying_price_and_time_from_url(self.html2)
-        self.assert_(isinstance(price, (int, float, complex)))
-        self.assert_(isinstance(quote_time, (datetime, Timestamp)))
+        self.assertIsInstance(price, (int, float, complex))
+        self.assertIsInstance(quote_time, (datetime, Timestamp))
 
     @network
     def test_sample_page_chg_float(self):
@@ -452,17 +452,17 @@ def test_is_s3_url(self):
     @network
     def test_read_yahoo(self):
         gs = DataReader("GS", "yahoo")
-        assert isinstance(gs, DataFrame)
+        self.assertIsInstance(gs, DataFrame)
 
     @network
     def test_read_google(self):
         gs = DataReader("GS", "google")
-        assert isinstance(gs, DataFrame)
+        self.assertIsInstance(gs, DataFrame)
 
     @network
     def test_read_fred(self):
         vix = DataReader("VIXCLS", "fred")
-        assert isinstance(vix, DataFrame)
+        self.assertIsInstance(vix, DataFrame)
 
     @network
     def test_read_famafrench(self):
@@ -470,8 +470,8 @@ def test_read_famafrench(self):
                      "F-F_Research_Data_Factors_weekly", "6_Portfolios_2x3",
                      "F-F_ST_Reversal_Factor", "F-F_Momentum_Factor"):
             ff = DataReader(name, "famafrench")
-            assert ff
-            assert isinstance(ff, dict)
+            self.assertTrue(ff is not None)
+            self.assertIsInstance(ff, dict)
 
 
 class TestFred(tm.TestCase):
@@ -498,7 +498,7 @@ def test_fred_nan(self):
         start = datetime(2010, 1, 1)
         end = datetime(2013, 1, 27)
         df = web.DataReader("DFII5", "fred", start, end)
-        assert pd.isnull(df.ix['2010-01-01'][0])
+        self.assertTrue(pd.isnull(df.ix['2010-01-01'][0]))
 
     @network
     def test_fred_parts(self):
@@ -510,7 +510,7 @@ def test_fred_parts(self):
         self.assertEqual(df.ix['2010-05-01'][0], 217.23)
 
         t = df.CPIAUCSL.values
-        assert np.issubdtype(t.dtype, np.floating)
+        self.assertTrue(np.issubdtype(t.dtype, np.floating))
         self.assertEqual(t.shape, (37,))
 
     @network
diff --git a/pandas/tests/test_categorical.py b/pandas/tests/test_categorical.py
--- a/pandas/tests/test_categorical.py
+++ b/pandas/tests/test_categorical.py
@@ -2423,6 +2423,39 @@ def f():
             df.append(df_wrong_categories)
         self.assertRaises(ValueError, f)
 
+
+    def test_merge(self):
+        # GH 9426
+
+        right = DataFrame({'c': {0: 'a', 1: 'b', 2: 'c', 3: 'd', 4: 'e'},
+                              'd': {0: 'null', 1: 'null', 2: 'null', 3: 'null', 4: 'null'}})
+        left = DataFrame({'a': {0: 'f', 1: 'f', 2: 'f', 3: 'f', 4: 'f'},
+                          'b': {0: 'g', 1: 'g', 2: 'g', 3: 'g', 4: 'g'}})
+        df = pd.merge(left, right, how='left', left_on='b', right_on='c')
+
+        # object-object
+        expected = df.copy()
+
+        # object-cat
+        cright = right.copy()
+        cright['d'] = cright['d'].astype('category')
+        result = pd.merge(left, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
+        # cat-object
+        cleft = left.copy()
+        cleft['b'] = cleft['b'].astype('category')
+        result = pd.merge(cleft, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
+        # cat-cat
+        cright = right.copy()
+        cright['d'] = cright['d'].astype('category')
+        cleft = left.copy()
+        cleft['b'] = cleft['b'].astype('category')
+        result = pd.merge(cleft, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
     def test_na_actions(self):
 
         cat = pd.Categorical([1,2,3,np.nan], categories=[1,2,3])
diff --git a/pandas/tseries/tests/test_tslib.py b/pandas/tseries/tests/test_tslib.py
--- a/pandas/tseries/tests/test_tslib.py
+++ b/pandas/tseries/tests/test_tslib.py
@@ -167,7 +167,7 @@ def test_repr(self):
 
         # dateutil zone change (only matters for repr)
         import dateutil
-        if dateutil.__version__ >= LooseVersion('2.3'):
+        if dateutil.__version__ >= LooseVersion('2.3') and dateutil.__version__ <= LooseVersion('2.4'):
             timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern', 'dateutil/US/Pacific']
         else:
             timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern', 'dateutil/America/Los_Angeles']