fix lexsort not working on object dtype for numpy version earlier than 1.11.0

add comment for using argsort with order argument instead of lexsort

Author: peterpanmj

pandas/_libs/algos_rank_helper.pxi.in

@@ -44,26 +44,24 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
 
         {{if dtype == 'object'}}
         ndarray sorted_data, values
-        ndarray[np.uint8_t, cast=True] sorted_namask
-        {{elif dtype != 'uint64'}}
-        ndarray[{{ctype}}] sorted_data, values
-        ndarray[np.uint8_t, cast=True] sorted_namask
         {{else}}
         ndarray[{{ctype}}] sorted_data, values
         {{endif}}
 
         ndarray[float64_t] ranks
         ndarray[int64_t] argsorted
+        ndarray[np.uint8_t, cast=True] sorted_mask
 
         {{if dtype == 'uint64'}}
         {{ctype}} val
         {{else}}
-        {{ctype}} val, nan_value, isnan
+        {{ctype}} val, nan_value
         {{endif}}
 
         float64_t sum_ranks = 0
         int tiebreak = 0
         bint keep_na = 0
+        bint isnan
         float count = 0.0
     tiebreak = tiebreakers[ties_method]
 
@@ -95,14 +93,24 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
     {{endif}}
 
     np.putmask(values, mask, nan_value)
+    {{else}}
+    mask = np.zeros(shape=len(values), dtype=bool)
     {{endif}}
 
     n = len(values)
     ranks = np.empty(n, dtype='f8')
 
     {{if dtype == 'object'}}
     try:
-        _as = values.argsort()
+        # lexsort on object array will raise TypeError for numpy version 
+        # earlier than 1.11.0
+        _dt = [('values', 'object'), ('mask', 'bool')]
+        _values = np.asarray(list(zip(mask, values)), dtype=_dt)
+        if ascending ^ (na_option == 'top'):
+            _values = np.asarray(list(zip(values, mask)), dtype=_dt)
+        else:
+            _values = np.asarray(list(zip(values, (~mask))), dtype=_dt)
+        _as = np.argsort(_values, kind='mergesort', order=('mask', 'values'))
     except TypeError:
         if not retry:
             raise
@@ -116,40 +124,37 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
     {{else}}
     if tiebreak == TIEBREAK_FIRST:
         # need to use a stable sort here
-        _as = values.argsort(kind='mergesort')
+        _as = np.lexsort(keys=(mask, values))
         if not ascending:
             tiebreak = TIEBREAK_FIRST_DESCENDING
     else:
-        _as = values.argsort()
+        _as = np.lexsort(keys=(mask, values))
     {{endif}}
 
     if not ascending:
         _as = _as[::-1]
 
     sorted_data = values.take(_as)
     # need to distinguish between pos/neg nan and real nan when keep_na is true
-    {{if dtype != 'uint64'}}
-    sorted_namask = mask.take(_as)
-    sorted_namask = sorted_namask.astype(np.bool)
-    {{endif}}
+    sorted_mask = mask.take(_as)
     argsorted = _as.astype('i8')
 
     {{if dtype == 'object'}}
     for i in range(n):
         sum_ranks += i + 1
         dups += 1
-        isnan = sorted_namask[i]
+        isnan = sorted_mask[i]
         val = util.get_value_at(sorted_data, i)
 
         if isnan and keep_na:
             ranks[argsorted[i]] = nan
-            sum_ranks = dups = 0
             continue
 
         count += 1.0
 
         if (i == n - 1 or
-            are_diff(util.get_value_at(sorted_data, i + 1), val)):
+                are_diff(util.get_value_at(sorted_data, i + 1), val) or
+                sorted_mask[i + 1]):
             if tiebreak == TIEBREAK_AVERAGE:
                 for j in range(i - dups + 1, i + 1):
                     ranks[argsorted[j]] = sum_ranks / dups
@@ -178,16 +183,15 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
             val = sorted_data[i]
 
             {{if dtype != 'uint64'}}
-            isnan = sorted_namask[i]
-            if  isnan and keep_na:
+            isnan = sorted_mask[i]
+            if isnan and keep_na:
                 ranks[argsorted[i]] = nan
-                sum_ranks = dups = 0
                 continue
             {{endif}}
 
             count += 1.0
 
-            if i == n - 1 or sorted_data[i + 1] != val:
+            if i == n - 1 or sorted_data[i + 1] != val or sorted_mask[i + 1]:
                 if tiebreak == TIEBREAK_AVERAGE:
                     for j in range(i - dups + 1, i + 1):
                         ranks[argsorted[j]] = sum_ranks / dups