fix v2

Author: OXPHOS

pandas/_libs/hashtable_class_helper.pxi.in

@@ -347,9 +347,10 @@ cdef class {{name}}HashTable(HashTable):
             for i in range(n):
                 val = values[i]
 
-                if check_null and {{null_condition}}:
-                    labels[i] = na_sentinel
-                    continue
+                if dropna:
+                    if check_null and {{null_condition}}:
+                        labels[i] = na_sentinel
+                        continue
 
                 k = kh_get_{{dtype}}(self.table, val)
 
@@ -830,13 +831,10 @@ cdef class PyObjectHashTable(HashTable):
             val = values[i]
             hash(val)
 
-            if check_null and val != val:
-                labels[i] = na_sentinel
-                continue
-
-            if dropna and val is None:
-                labels[i] = na_sentinel
-                continue
+            if dropna:
+                if (check_null and val != val) or val is None:
+                    labels[i] = na_sentinel
+                    continue
 
             k = kh_get_pymap(self.table, <PyObject*>val)
             if k != self.table.n_buckets:

pandas/core/algorithms.py

@@ -520,7 +520,7 @@ def sort_mixed(values):
 
 
 def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None,
-              dropna=False):
+              dropna=True):
     """
     Encode input values as an enumerated type or categorical variable
 

pandas/core/categorical.py

@@ -234,7 +234,8 @@ class Categorical(PandasObject):
     __array_priority__ = 1000
     _typ = 'categorical'
 
-    def __init__(self, values, categories=None, ordered=False, fastpath=False):
+    def __init__(self, values, categories=None, ordered=False, fastpath=False,
+                 dropna=True):
 
         self._validate_ordered(ordered)
 
@@ -281,9 +282,10 @@ def __init__(self, values, categories=None, ordered=False, fastpath=False):
 
         if categories is None:
             try:
-                codes, categories = factorize(values, sort=True)
+                codes, categories = factorize(values, sort=True, dropna=dropna)
             except TypeError:
-                codes, categories = factorize(values, sort=False)
+                codes, categories = factorize(values, sort=False,
+                                              dropna=dropna)
                 if ordered:
                     # raise, as we don't have a sortable data structure and so
                     # the user should give us one by specifying categories
@@ -2106,7 +2108,7 @@ def _convert_to_list_like(list_like):
         return [list_like]
 
 
-def _factorize_from_iterable(values):
+def _factorize_from_iterable(values, dropna=True):
     """
     Factorize an input `values` into `categories` and `codes`. Preserves
     categorical dtype in `categories`.
@@ -2137,13 +2139,13 @@ def _factorize_from_iterable(values):
                                       ordered=values.ordered)
         codes = values.codes
     else:
-        cat = Categorical(values, ordered=True)
+        cat = Categorical(values, ordered=True, dropna=dropna)
         categories = cat.categories
         codes = cat.codes
     return codes, categories
 
 
-def _factorize_from_iterables(iterables):
+def _factorize_from_iterables(iterables, dropna=True):
     """
     A higher-level wrapper over `_factorize_from_iterable`.
 
@@ -2165,4 +2167,5 @@ def _factorize_from_iterables(iterables):
     if len(iterables) == 0:
         # For consistency, it should return a list of 2 lists.
         return [[], []]
-    return map(list, lzip(*[_factorize_from_iterable(it) for it in iterables]))
+    return map(list, lzip(*[_factorize_from_iterable(it, dropna)
+                            for it in iterables]))

pandas/core/indexes/multi.py

@@ -1043,7 +1043,7 @@ def lexsort_depth(self):
         return 0
 
     @classmethod
-    def from_arrays(cls, arrays, sortorder=None, names=None):
+    def from_arrays(cls, arrays, sortorder=None, names=None, dropna=False):
         """
         Convert arrays to MultiIndex
 
@@ -1083,7 +1083,7 @@ def from_arrays(cls, arrays, sortorder=None, names=None):
 
         from pandas.core.categorical import _factorize_from_iterables
 
-        labels, levels = _factorize_from_iterables(arrays)
+        labels, levels = _factorize_from_iterables(arrays, dropna=dropna)
         if names is None:
             names = [getattr(arr, "name", None) for arr in arrays]
 

pandas/tests/reshape/test_pivot.py

@@ -1180,17 +1180,18 @@ def test_margin_dropna(self):
         df = pd.DataFrame({'a': [1, 2, 2, 2, 2, np.nan],
                            'b': [3, 3, 4, 4, 4, 4]})
         actual = pd.crosstab(df.a, df.b, margins=True, dropna=False)
-        expected = pd.DataFrame([[1, 0, 1], [1, 3, 4], [2, 4, 6]])
-        expected.index = Index([1.0, 2.0, 'All'], name='a')
+        expected = pd.DataFrame([[1, 0, 1], [1, 3, 4], [0, 1, 1], [2, 4, 6]])
+        expected.index = Index([1.0, 2.0, np.nan, 'All'], name='a')
         expected.columns = Index([3, 4, 'All'], name='b')
         tm.assert_frame_equal(actual, expected)
 
         df = DataFrame({'a': [1, np.nan, np.nan, np.nan, 2, np.nan],
                         'b': [3, np.nan, 4, 4, 4, 4]})
         actual = pd.crosstab(df.a, df.b, margins=True, dropna=False)
-        expected = pd.DataFrame([[1, 0, 1], [0, 1, 1], [1, 4, 6]])
-        expected.index = Index([1.0, 2.0, 'All'], name='a')
-        expected.columns = Index([3.0, 4.0, 'All'], name='b')
+        expected = pd.DataFrame([[1, 0, 0, 1], [0, 1, 0, 1], [0, 3, 1, 4],
+                                 [1, 4, 0, 6]])
+        expected.index = Index([1.0, 2.0, np.nan, 'All'], name='a')
+        expected.columns = Index([3.0, 4.0, np.nan, 'All'], name='b')
         tm.assert_frame_equal(actual, expected)
 
         a = np.array(['foo', 'foo', 'foo', 'bar',
@@ -1202,21 +1203,25 @@ def test_margin_dropna(self):
 
         actual = pd.crosstab(a, [b, c], rownames=['a'],
                              colnames=['b', 'c'], margins=True, dropna=False)
-        m = MultiIndex.from_arrays([['one', 'one', 'two', 'two', 'All'],
-                                    ['dull', 'shiny', 'dull', 'shiny', '']],
+        m = MultiIndex.from_arrays([[np.nan, np.nan, 'one', 'one', 'two',
+                                     'two', 'All'],
+                                    ['dull', 'shiny', 'dull', 'shiny', 'dull',
+                                     'shiny', '']],
                                    names=['b', 'c'])
-        expected = DataFrame([[1, 0, 1, 0, 2], [2, 0, 1, 1, 5],
-                              [3, 0, 2, 1, 7]], columns=m)
+        expected = DataFrame([[0, 0, 1, 0, 1, 0, 2], [0, 1, 2, 0, 1, 1, 5],
+                              [0, 1, 3, 0, 2, 1, 7]], columns=m)
         expected.index = Index(['bar', 'foo', 'All'], name='a')
         tm.assert_frame_equal(actual, expected)
 
         actual = pd.crosstab([a, b], c, rownames=['a', 'b'],
                              colnames=['c'], margins=True, dropna=False)
-        m = MultiIndex.from_arrays([['bar', 'bar', 'foo', 'foo', 'All'],
-                                    ['one', 'two', 'one', 'two', '']],
+        m = MultiIndex.from_arrays([['bar', 'bar', 'bar', 'foo', 'foo',
+                                     'foo', 'All'],
+                                    [np.nan, 'one', 'two', np.nan, 'one',
+                                     'two', '']],
                                    names=['a', 'b'])
-        expected = DataFrame([[1, 0, 1], [1, 0, 1], [2, 0, 2], [1, 1, 2],
-                              [5, 2, 7]], index=m)
+        expected = DataFrame([[0, 0, 0], [1, 0, 1], [1, 0, 1], [0, 1, 1],
+                              [2, 0, 2], [1, 1, 2], [5, 2, 7]], index=m)
         expected.columns = Index(['dull', 'shiny', 'All'], name='c')
         tm.assert_frame_equal(actual, expected)