catch complex nan in util.is_nan, de-dup+optimize libmissing, tests (#24628)

Author: jbrockmendel

pandas/_libs/missing.pyx

@@ -12,33 +12,16 @@ cimport pandas._libs.util as util
 
 from pandas._libs.tslibs.np_datetime cimport (
     get_timedelta64_value, get_datetime64_value)
-from pandas._libs.tslibs.nattype cimport checknull_with_nat, c_NaT
+from pandas._libs.tslibs.nattype cimport (
+    checknull_with_nat, c_NaT as NaT, is_null_datetimelike)
+
 
 cdef float64_t INF = <float64_t>np.inf
 cdef float64_t NEGINF = -INF
 
 cdef int64_t NPY_NAT = util.get_nat()
 
 
-cdef inline bint _check_all_nulls(object val):
-    """ utility to check if a value is any type of null """
-    res: bint
-
-    if isinstance(val, (float, complex)):
-        res = val != val
-    elif val is c_NaT:
-        res = 1
-    elif val is None:
-        res = 1
-    elif util.is_datetime64_object(val):
-        res = get_datetime64_value(val) == NPY_NAT
-    elif util.is_timedelta64_object(val):
-        res = get_timedelta64_value(val) == NPY_NAT
-    else:
-        res = 0
-    return res
-
-
 cpdef bint checknull(object val):
     """
     Return boolean describing of the input is NA-like, defined here as any
@@ -62,18 +45,7 @@ cpdef bint checknull(object val):
     The difference between `checknull` and `checknull_old` is that `checknull`
     does *not* consider INF or NEGINF to be NA.
     """
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val  # and val != INF and val != NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is c_NaT:
-        return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif util.is_array(val):
-        return False
-    else:
-        return val is None or util.is_nan(val)
+    return is_null_datetimelike(val, inat_is_null=False)
 
 
 cpdef bint checknull_old(object val):
@@ -101,18 +73,11 @@ cpdef bint checknull_old(object val):
     The difference between `checknull` and `checknull_old` is that `checknull`
     does *not* consider INF or NEGINF to be NA.
     """
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val or val == INF or val == NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is c_NaT:
+    if checknull(val):
         return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif util.is_array(val):
-        return False
-    else:
-        return val is None or util.is_nan(val)
+    elif util.is_float_object(val) or util.is_complex_object(val):
+        return val == INF or val == NEGINF
+    return False
 
 
 cdef inline bint _check_none_nan_inf_neginf(object val):
@@ -128,7 +93,7 @@ cdef inline bint _check_none_nan_inf_neginf(object val):
 cpdef ndarray[uint8_t] isnaobj(ndarray arr):
     """
     Return boolean mask denoting which elements of a 1-D array are na-like,
-    according to the criteria defined in `_check_all_nulls`:
+    according to the criteria defined in `checknull`:
      - None
      - nan
      - NaT
@@ -154,7 +119,7 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr):
     result = np.empty(n, dtype=np.uint8)
     for i in range(n):
         val = arr[i]
-        result[i] = _check_all_nulls(val)
+        result[i] = checknull(val)
     return result.view(np.bool_)
 
 
@@ -189,7 +154,7 @@ def isnaobj_old(ndarray arr):
     result = np.zeros(n, dtype=np.uint8)
     for i in range(n):
         val = arr[i]
-        result[i] = val is c_NaT or _check_none_nan_inf_neginf(val)
+        result[i] = val is NaT or _check_none_nan_inf_neginf(val)
     return result.view(np.bool_)
 
 
@@ -299,7 +264,7 @@ cdef inline bint is_null_datetime64(v):
     if checknull_with_nat(v):
         return True
     elif util.is_datetime64_object(v):
-        return v.view('int64') == NPY_NAT
+        return get_datetime64_value(v) == NPY_NAT
     return False
 
 
@@ -309,7 +274,7 @@ cdef inline bint is_null_timedelta64(v):
     if checknull_with_nat(v):
         return True
     elif util.is_timedelta64_object(v):
-        return v.view('int64') == NPY_NAT
+        return get_timedelta64_value(v) == NPY_NAT
     return False
 
 

pandas/_libs/tslibs/nattype.pxd

@@ -17,4 +17,4 @@ cdef _NaT c_NaT
 
 
 cdef bint checknull_with_nat(object val)
-cpdef bint is_null_datetimelike(object val)
+cpdef bint is_null_datetimelike(object val, bint inat_is_null=*)

pandas/_libs/tslibs/nattype.pyx

@@ -14,6 +14,8 @@ cimport numpy as cnp
 from numpy cimport int64_t
 cnp.import_array()
 
+from pandas._libs.tslibs.np_datetime cimport (
+    get_datetime64_value, get_timedelta64_value)
 cimport pandas._libs.tslibs.util as util
 from pandas._libs.tslibs.util cimport (
     get_nat, is_integer_object, is_float_object, is_datetime64_object,
@@ -686,26 +688,30 @@ cdef inline bint checknull_with_nat(object val):
     return val is None or util.is_nan(val) or val is c_NaT
 
 
-cpdef bint is_null_datetimelike(object val):
+cpdef bint is_null_datetimelike(object val, bint inat_is_null=True):
     """
     Determine if we have a null for a timedelta/datetime (or integer versions)
 
     Parameters
     ----------
     val : object
+    inat_is_null : bool, default True
+        Whether to treat integer iNaT value as null
 
     Returns
     -------
     null_datetimelike : bool
     """
-    if val is None or util.is_nan(val):
+    if val is None:
         return True
     elif val is c_NaT:
         return True
+    elif util.is_float_object(val) or util.is_complex_object(val):
+        return val != val
     elif util.is_timedelta64_object(val):
-        return val.view('int64') == NPY_NAT
+        return get_timedelta64_value(val) == NPY_NAT
     elif util.is_datetime64_object(val):
-        return val.view('int64') == NPY_NAT
-    elif util.is_integer_object(val):
+        return get_datetime64_value(val) == NPY_NAT
+    elif inat_is_null and util.is_integer_object(val):
         return val == NPY_NAT
     return False

pandas/_libs/tslibs/util.pxd

@@ -215,7 +215,8 @@ cdef inline bint is_offset_object(object val):
 
 cdef inline bint is_nan(object val):
     """
-    Check if val is a Not-A-Number float, including float('NaN') and np.nan.
+    Check if val is a Not-A-Number float or complex, including
+    float('NaN') and np.nan.
 
     Parameters
     ----------
@@ -225,4 +226,4 @@ cdef inline bint is_nan(object val):
     -------
     is_nan : bool
     """
-    return is_float_object(val) and val != val
+    return (is_float_object(val) or is_complex_object(val)) and val != val

pandas/tests/dtypes/test_missing.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
 
 from datetime import datetime
+from decimal import Decimal
 from warnings import catch_warnings, filterwarnings, simplefilter
 
 import numpy as np
 import pytest
 
 from pandas._libs import missing as libmissing
-from pandas._libs.tslib import iNaT
+from pandas._libs.tslibs import iNaT, is_null_datetimelike
 from pandas.compat import u
 
 from pandas.core.dtypes.common import is_scalar
@@ -392,3 +393,106 @@ def test_empty_like(self):
         expected = np.array([True])
 
         self._check_behavior(arr, expected)
+
+
+m8_units = ['as', 'ps', 'ns', 'us', 'ms', 's',
+            'm', 'h', 'D', 'W', 'M', 'Y']
+
+na_vals = [
+    None,
+    NaT,
+    float('NaN'),
+    complex('NaN'),
+    np.nan,
+    np.float64('NaN'),
+    np.float32('NaN'),
+    np.complex64(np.nan),
+    np.complex128(np.nan),
+    np.datetime64('NaT'),
+    np.timedelta64('NaT'),
+] + [
+    np.datetime64('NaT', unit) for unit in m8_units
+] + [
+    np.timedelta64('NaT', unit) for unit in m8_units
+]
+
+inf_vals = [
+    float('inf'),
+    float('-inf'),
+    complex('inf'),
+    complex('-inf'),
+    np.inf,
+    np.NINF,
+]
+
+int_na_vals = [
+    # Values that match iNaT, which we treat as null in specific cases
+    np.int64(NaT.value),
+    int(NaT.value),
+]
+
+sometimes_na_vals = [
+    Decimal('NaN'),
+]
+
+never_na_vals = [
+    # float/complex values that when viewed as int64 match iNaT
+    -0.0,
+    np.float64('-0.0'),
+    -0j,
+    np.complex64(-0j),
+]
+
+
+class TestLibMissing(object):
+    def test_checknull(self):
+        for value in na_vals:
+            assert libmissing.checknull(value)
+
+        for value in inf_vals:
+            assert not libmissing.checknull(value)
+
+        for value in int_na_vals:
+            assert not libmissing.checknull(value)
+
+        for value in sometimes_na_vals:
+            assert not libmissing.checknull(value)
+
+        for value in never_na_vals:
+            assert not libmissing.checknull(value)
+
+    def checknull_old(self):
+        for value in na_vals:
+            assert libmissing.checknull_old(value)
+
+        for value in inf_vals:
+            assert libmissing.checknull_old(value)
+
+        for value in int_na_vals:
+            assert not libmissing.checknull_old(value)
+
+        for value in sometimes_na_vals:
+            assert not libmissing.checknull_old(value)
+
+        for value in never_na_vals:
+            assert not libmissing.checknull_old(value)
+
+    def test_is_null_datetimelike(self):
+        for value in na_vals:
+            assert is_null_datetimelike(value)
+            assert is_null_datetimelike(value, False)
+
+        for value in inf_vals:
+            assert not is_null_datetimelike(value)
+            assert not is_null_datetimelike(value, False)
+
+        for value in int_na_vals:
+            assert is_null_datetimelike(value)
+            assert not is_null_datetimelike(value, False)
+
+        for value in sometimes_na_vals:
+            assert not is_null_datetimelike(value)
+            assert not is_null_datetimelike(value, False)
+
+        for value in never_na_vals:
+            assert not is_null_datetimelike(value)