BUG: Regression in merging Categorical and object dtypes (GH9426)

Author: jreback

doc/source/whatsnew/v0.16.0.txt

@@ -521,7 +521,7 @@ Bug Fixes
 
 
 - ``SparseSeries`` and ``SparsePanel`` now accept zero argument constructors (same as their non-sparse counterparts) (:issue:`9272`).
-
+- Regression in merging Categoricals and object dtypes (:issue:`9426`)
 - Bug in ``read_csv`` with buffer overflows with certain malformed input files (:issue:`9205`)
 - Bug in groupby MultiIndex with missing pair (:issue:`9049`, :issue:`9344`)
 - Fixed bug in ``Series.groupby`` where grouping on ``MultiIndex`` levels would ignore the sort argument (:issue:`9444`)

pandas/core/common.py

@@ -1146,7 +1146,9 @@ def _maybe_promote(dtype, fill_value=np.nan):
         dtype = np.object_
 
     # in case we have a string that looked like a number
-    if issubclass(np.dtype(dtype).type, compat.string_types):
+    if is_categorical_dtype(dtype):
+        dtype = dtype
+    elif issubclass(np.dtype(dtype).type, compat.string_types):
         dtype = np.object_
 
     return dtype, fill_value

pandas/core/internals.py

@@ -4327,8 +4327,9 @@ def dtype(self):
         if not self.needs_filling:
             return self.block.dtype
         else:
-            return np.dtype(com._maybe_promote(self.block.dtype,
-                                               self.block.fill_value)[0])
+            return com._get_dtype(com._maybe_promote(self.block.dtype,
+                                                     self.block.fill_value)[0])
+
         return self._dtype
 
     @cache_readonly

pandas/tests/test_categorical.py

@@ -2423,6 +2423,39 @@ def f():
             df.append(df_wrong_categories)
         self.assertRaises(ValueError, f)
 
+
+    def test_merge(self):
+        # GH 9426
+
+        right = DataFrame({'c': {0: 'a', 1: 'b', 2: 'c', 3: 'd', 4: 'e'},
+                              'd': {0: 'null', 1: 'null', 2: 'null', 3: 'null', 4: 'null'}})
+        left = DataFrame({'a': {0: 'f', 1: 'f', 2: 'f', 3: 'f', 4: 'f'},
+                          'b': {0: 'g', 1: 'g', 2: 'g', 3: 'g', 4: 'g'}})
+        df = pd.merge(left, right, how='left', left_on='b', right_on='c')
+
+        # object-object
+        expected = df.copy()
+
+        # object-cat
+        cright = right.copy()
+        cright['d'] = cright['d'].astype('category')
+        result = pd.merge(left, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
+        # cat-object
+        cleft = left.copy()
+        cleft['b'] = cleft['b'].astype('category')
+        result = pd.merge(cleft, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
+        # cat-cat
+        cright = right.copy()
+        cright['d'] = cright['d'].astype('category')
+        cleft = left.copy()
+        cleft['b'] = cleft['b'].astype('category')
+        result = pd.merge(cleft, cright, how='left', left_on='b', right_on='c')
+        tm.assert_frame_equal(result, expected)
+
     def test_na_actions(self):
 
         cat = pd.Categorical([1,2,3,np.nan], categories=[1,2,3])