types, cleanup, let cython handle dtype dispatch

Author: jbrockmendel

pandas/_libs/groupby_helper.pxi.in

@@ -735,11 +735,6 @@ def group_min(ndarray[groupby_t, ndim=2] out,
                     out[i, j] = minx[i, j]
 
 
-group_min_float64 = group_min["float64_t"]
-group_min_float32 = group_min["float32_t"]
-group_min_int64 = group_min["int64_t"]
-
-
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_cummin(groupby_t[:, :] out,
@@ -788,11 +783,6 @@ def group_cummin(groupby_t[:, :] out,
                         out[i, j] = mval
 
 
-group_cummin_float64 = group_cummin["float64_t"]
-group_cummin_float32 = group_cummin["float32_t"]
-group_cummin_int64 = group_cummin["int64_t"]
-
-
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_cummax(groupby_t[:, :] out,
@@ -838,8 +828,3 @@ def group_cummax(groupby_t[:, :] out,
                         if val > mval:
                             accum[lab, j] = mval = val
                         out[i, j] = mval
-
-
-group_cummax_float64 = group_cummax["float64_t"]
-group_cummax_float32 = group_cummax["float32_t"]
-group_cummax_int64 = group_cummax["int64_t"]

pandas/_libs/lib.pyx

@@ -263,10 +263,10 @@ def fast_unique_multiple_list_gen(object gen, bint sort=True):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def dicts_to_array(list dicts, list columns):
+def dicts_to_array(dicts: list, columns: list):
     cdef:
         Py_ssize_t i, j, k, n
-        ndarray[object, ndim=2] result
+        object[:, :] result
         dict row
         object col, onan = np.nan
 
@@ -284,7 +284,7 @@ def dicts_to_array(list dicts, list columns):
             else:
                 result[i, j] = onan
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 def fast_zip(list ndarrays):
@@ -343,17 +343,17 @@ def get_reverse_indexer(int64_t[:] indexer, Py_ssize_t length):
 
     cdef:
         Py_ssize_t i, n = len(indexer)
-        ndarray[int64_t] rev_indexer
+        int64_t[:] rev_indexer
         int64_t idx
 
     rev_indexer = np.empty(length, dtype=np.int64)
-    rev_indexer.fill(-1)
+    rev_indexer[:] = -1
     for i in range(n):
         idx = indexer[i]
         if idx != -1:
             rev_indexer[idx] = i
 
-    return rev_indexer
+    return rev_indexer.base  # `.base` to access underlying np.ndarray
 
 
 @cython.wraparound(False)
@@ -460,7 +460,7 @@ def maybe_booleans_to_slice(ndarray[uint8_t] mask):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def array_equivalent_object(left: object[:], right: object[:]) -> bint:
+def array_equivalent_object(left: object[:], right: object[:]) -> bool:
     """ perform an element by element comparion on 1-d object arrays
         taking into account nan positions """
     cdef:
@@ -484,7 +484,7 @@ def array_equivalent_object(left: object[:], right: object[:]) -> bint:
 def astype_intsafe(object[:] arr, new_dtype):
     cdef:
         Py_ssize_t i, n = len(arr)
-        object v
+        object val
         bint is_datelike
         ndarray result
 
@@ -493,11 +493,11 @@ def astype_intsafe(object[:] arr, new_dtype):
 
     result = np.empty(n, dtype=new_dtype)
     for i in range(n):
-        v = arr[i]
-        if is_datelike and checknull(v):
+        val = arr[i]
+        if is_datelike and checknull(val):
             result[i] = NPY_NAT
         else:
-            result[i] = v
+            result[i] = val
 
     return result
 
@@ -524,7 +524,7 @@ def astype_unicode(arr: ndarray, skipna: bool=False) -> ndarray[object]:
     cdef:
         object arr_i
         Py_ssize_t i, n = arr.size
-        ndarray[object] result = np.empty(n, dtype=object)
+        object[:] result = np.empty(n, dtype=object)
 
     for i in range(n):
         arr_i = arr[i]
@@ -534,7 +534,7 @@ def astype_unicode(arr: ndarray, skipna: bool=False) -> ndarray[object]:
 
         result[i] = arr_i
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 @cython.wraparound(False)
@@ -559,7 +559,7 @@ def astype_str(arr: ndarray, skipna: bool = False) -> ndarray[object]:
     cdef:
         object arr_i
         Py_ssize_t i, n = arr.size
-        ndarray[object] result = np.empty(n, dtype=object)
+        object[:] result = np.empty(n, dtype=object)
 
     for i in range(n):
         arr_i = arr[i]
@@ -569,24 +569,24 @@ def astype_str(arr: ndarray, skipna: bool = False) -> ndarray[object]:
 
         result[i] = arr_i
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def clean_index_list(list obj):
+def clean_index_list(obj: list):
     """
     Utility used in pandas.core.index.ensure_index
     """
     cdef:
         Py_ssize_t i, n = len(obj)
-        object v
+        object val
         bint all_arrays = 1
 
     for i in range(n):
-        v = obj[i]
-        if not (isinstance(v, list) or
-                util.is_array(v) or hasattr(v, '_data')):
+        val = obj[i]
+        if not (isinstance(val, list) or
+                util.is_array(val) or hasattr(val, '_data')):
             all_arrays = 0
             break
 
@@ -595,11 +595,9 @@ def clean_index_list(list obj):
 
     # don't force numpy coerce with nan's
     inferred = infer_dtype(obj)
-    if inferred in ['string', 'bytes', 'unicode',
-                    'mixed', 'mixed-integer']:
+    if inferred in ['string', 'bytes', 'unicode', 'mixed', 'mixed-integer']:
         return np.asarray(obj, dtype=object), 0
     elif inferred in ['integer']:
-
         # TODO: we infer an integer but it *could* be a uint64
         try:
             return np.asarray(obj, dtype='int64'), 0
@@ -680,13 +678,13 @@ def generate_bins_dt64(ndarray[int64_t] values, int64_t[:] binner,
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def row_bool_subset(ndarray[float64_t, ndim=2] values,
+def row_bool_subset(float64_t[:, :] values,
                     ndarray[uint8_t, cast=True] mask):
     cdef:
         Py_ssize_t i, j, n, k, pos = 0
-        ndarray[float64_t, ndim=2] out
+        float64_t[:, :] out
 
-    n, k = (<object> values).shape
+    n, k = (<object>values).shape
     assert (n == len(mask))
 
     out = np.empty((mask.sum(), k), dtype=np.float64)
@@ -697,7 +695,7 @@ def row_bool_subset(ndarray[float64_t, ndim=2] values,
                 out[pos, j] = values[i, j]
             pos += 1
 
-    return out
+    return out.base  # `.base` to access underlying np.ndarray
 
 
 @cython.boundscheck(False)
@@ -706,7 +704,7 @@ def row_bool_subset_object(object[:, :] values,
                            ndarray[uint8_t, cast=True] mask):
     cdef:
         Py_ssize_t i, j, n, k, pos = 0
-        ndarray[object, ndim=2] out
+        object[:, :] out
 
     n, k = (<object>values).shape
     assert (n == len(mask))
@@ -719,7 +717,7 @@ def row_bool_subset_object(object[:, :] values,
                 out[pos, j] = values[i, j]
             pos += 1
 
-    return out
+    return out.base  # `.base` to access underlying np.ndarray
 
 
 @cython.boundscheck(False)
@@ -846,19 +844,19 @@ def indices_fast(object index, int64_t[:] labels, list keys,
 
 # core.common import for fast inference checks
 
-def is_float(obj: object) -> bint:
+def is_float(obj: object) -> bool:
     return util.is_float_object(obj)
 
 
-def is_integer(obj: object) -> bint:
+def is_integer(obj: object) -> bool:
     return util.is_integer_object(obj)
 
 
-def is_bool(obj: object) -> bint:
+def is_bool(obj: object) -> bool:
     return util.is_bool_object(obj)
 
 
-def is_complex(obj: object) -> bint:
+def is_complex(obj: object) -> bool:
     return util.is_complex_object(obj)
 
 
@@ -870,7 +868,7 @@ cpdef bint is_interval(object obj):
     return getattr(obj, '_typ', '_typ') == 'interval'
 
 
-def is_period(val: object) -> bint:
+def is_period(val: object) -> bool:
     """ Return a boolean if this is a Period object """
     return util.is_period_object(val)
 
@@ -1352,7 +1350,7 @@ def infer_datetimelike_array(arr: object) -> object:
             seen_datetime = 1
         elif PyDate_Check(v):
             seen_date = 1
-        elif is_timedelta(v) or util.is_timedelta64_object(v):
+        elif is_timedelta(v):
             # timedelta, or timedelta64
             seen_timedelta = 1
         else:
@@ -1633,7 +1631,7 @@ cpdef bint is_datetime64_array(ndarray values):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def is_datetime_with_singletz_array(values: ndarray) -> bint:
+def is_datetime_with_singletz_array(values: ndarray) -> bool:
     """
     Check values have the same tzinfo attribute.
     Doesn't check values are datetime-like types.
@@ -2138,7 +2136,7 @@ def map_infer_mask(ndarray arr, object f, uint8_t[:] mask, bint convert=1):
     """
     cdef:
         Py_ssize_t i, n
-        ndarray[object] result
+        object[:] result
         object val
 
     n = len(arr)
@@ -2163,7 +2161,7 @@ def map_infer_mask(ndarray arr, object f, uint8_t[:] mask, bint convert=1):
                                      convert_datetime=0,
                                      convert_timedelta=0)
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 @cython.wraparound(False)
@@ -2208,7 +2206,7 @@ def map_infer(ndarray arr, object f, bint convert=1):
     return result.base  # `.base` to access underlying np.ndarray
 
 
-def to_object_array(list rows, int min_width=0):
+def to_object_array(rows: list, min_width: int = 0):
     """
     Convert a list of lists into an object array.
 
@@ -2228,7 +2226,7 @@ def to_object_array(list rows, int min_width=0):
     """
     cdef:
         Py_ssize_t i, j, n, k, tmp
-        ndarray[object, ndim=2] result
+        object[:, :] result
         list row
 
     n = len(rows)
@@ -2247,13 +2245,13 @@ def to_object_array(list rows, int min_width=0):
         for j in range(len(row)):
             result[i, j] = row[j]
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 def tuples_to_object_array(ndarray[object] tuples):
     cdef:
         Py_ssize_t i, j, n, k, tmp
-        ndarray[object, ndim=2] result
+        object[:, :] result
         tuple tup
 
     n = len(tuples)
@@ -2264,13 +2262,13 @@ def tuples_to_object_array(ndarray[object] tuples):
         for j in range(k):
             result[i, j] = tup[j]
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
-def to_object_array_tuples(list rows):
+def to_object_array_tuples(rows: list):
     cdef:
         Py_ssize_t i, j, n, k, tmp
-        ndarray[object, ndim=2] result
+        object[:, :] result
         tuple row
 
     n = len(rows)
@@ -2295,7 +2293,7 @@ def to_object_array_tuples(list rows):
             for j in range(len(row)):
                 result[i, j] = row[j]
 
-    return result
+    return result.base  # `.base` to access underlying np.ndarray
 
 
 @cython.wraparound(False)