From 80250aabbe2d5b1a9538bf2e8e2c656f195c6086 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Mon, 27 Apr 2020 18:54:49 -0500 Subject: [PATCH 1/3] REF/CLN: Parametrize _isna --- pandas/core/dtypes/missing.py | 52 ++++++++++++----------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 92e1b17c41694..a38967c3b7b63 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -1,6 +1,8 @@ """ missing types & inference """ +from functools import partial + import numpy as np from pandas._config import get_option @@ -124,61 +126,44 @@ def isna(obj): isnull = isna -def _isna_new(obj): - - if is_scalar(obj): - return libmissing.checknull(obj) - # hack (for now) because MI registers as ndarray - elif isinstance(obj, ABCMultiIndex): - raise NotImplementedError("isna is not defined for MultiIndex") - elif isinstance(obj, type): - return False - elif isinstance(obj, (ABCSeries, np.ndarray, ABCIndexClass, ABCExtensionArray)): - return _isna_ndarraylike(obj, old=False) - elif isinstance(obj, ABCDataFrame): - return obj.isna() - elif isinstance(obj, list): - return _isna_ndarraylike(np.asarray(obj, dtype=object), old=False) - elif hasattr(obj, "__array__"): - return _isna_ndarraylike(np.asarray(obj), old=False) - else: - return False - - -def _isna_old(obj): +def _isna(obj, old: bool = False): """ - Detect missing values, treating None, NaN, INF, -INF as null. + Detect missing values, treating None, NaN or NA as null. If old is True + infinite values will also be treated as null. Parameters ---------- arr: ndarray or object value + Input array or scalar value. + old: bool + Whether to treat infinity as null. Returns ------- boolean ndarray or boolean """ if is_scalar(obj): - return libmissing.checknull_old(obj) + if old: + return libmissing.checknull_old(obj) + else: + return libmissing.checknull(obj) # hack (for now) because MI registers as ndarray elif isinstance(obj, ABCMultiIndex): raise NotImplementedError("isna is not defined for MultiIndex") elif isinstance(obj, type): return False elif isinstance(obj, (ABCSeries, np.ndarray, ABCIndexClass, ABCExtensionArray)): - return _isna_ndarraylike(obj, old=True) + return _isna_ndarraylike(obj, old=old) elif isinstance(obj, ABCDataFrame): return obj.isna() elif isinstance(obj, list): - return _isna_ndarraylike(np.asarray(obj, dtype=object), old=True) + return _isna_ndarraylike(np.asarray(obj, dtype=object), old=old) elif hasattr(obj, "__array__"): - return _isna_ndarraylike(np.asarray(obj), old=True) + return _isna_ndarraylike(np.asarray(obj), old=old) else: return False -_isna = _isna_new - - def _use_inf_as_na(key): """ Option change callback for na/inf behaviour. @@ -200,11 +185,8 @@ def _use_inf_as_na(key): * https://stackoverflow.com/questions/4859217/ programmatically-creating-variables-in-python/4859312#4859312 """ - flag = get_option(key) - if flag: - globals()["_isna"] = _isna_old - else: - globals()["_isna"] = _isna_new + old = get_option(key) + globals()["_isna"] = partial(_isna, old=old) def _isna_ndarraylike(obj, old: bool = False): From c785a8534ac14ad6b1c22e374212b200619c9a4a Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Tue, 28 Apr 2020 11:44:25 -0500 Subject: [PATCH 2/3] old -> inf_as_na --- pandas/core/dtypes/missing.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index a38967c3b7b63..bdc9227909d90 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -126,7 +126,7 @@ def isna(obj): isnull = isna -def _isna(obj, old: bool = False): +def _isna(obj, inf_as_na: bool = False): """ Detect missing values, treating None, NaN or NA as null. If old is True infinite values will also be treated as null. @@ -135,7 +135,7 @@ def _isna(obj, old: bool = False): ---------- arr: ndarray or object value Input array or scalar value. - old: bool + inf_as_na: bool Whether to treat infinity as null. Returns @@ -143,7 +143,7 @@ def _isna(obj, old: bool = False): boolean ndarray or boolean """ if is_scalar(obj): - if old: + if inf_as_na: return libmissing.checknull_old(obj) else: return libmissing.checknull(obj) @@ -153,13 +153,13 @@ def _isna(obj, old: bool = False): elif isinstance(obj, type): return False elif isinstance(obj, (ABCSeries, np.ndarray, ABCIndexClass, ABCExtensionArray)): - return _isna_ndarraylike(obj, old=old) + return _isna_ndarraylike(obj, inf_as_na=inf_as_na) elif isinstance(obj, ABCDataFrame): return obj.isna() elif isinstance(obj, list): - return _isna_ndarraylike(np.asarray(obj, dtype=object), old=old) + return _isna_ndarraylike(np.asarray(obj, dtype=object), inf_as_na=inf_as_na) elif hasattr(obj, "__array__"): - return _isna_ndarraylike(np.asarray(obj), old=old) + return _isna_ndarraylike(np.asarray(obj), inf_as_na=inf_as_na) else: return False @@ -185,11 +185,11 @@ def _use_inf_as_na(key): * https://stackoverflow.com/questions/4859217/ programmatically-creating-variables-in-python/4859312#4859312 """ - old = get_option(key) - globals()["_isna"] = partial(_isna, old=old) + inf_as_na = get_option(key) + globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na) -def _isna_ndarraylike(obj, old: bool = False): +def _isna_ndarraylike(obj, inf_as_na: bool = False): """ Return an array indicating which values of the input array are NaN / NA. @@ -197,7 +197,7 @@ def _isna_ndarraylike(obj, old: bool = False): ---------- obj: array-like The input array whose elements are to be checked. - old: bool + inf_as_na: bool Whether or not to treat infinite values as NA. Returns @@ -209,17 +209,17 @@ def _isna_ndarraylike(obj, old: bool = False): dtype = values.dtype if is_extension_array_dtype(dtype): - if old: + if inf_as_na: result = values.isna() | (values == -np.inf) | (values == np.inf) else: result = values.isna() elif is_string_dtype(dtype): - result = _isna_string_dtype(values, dtype, old=old) + result = _isna_string_dtype(values, dtype, inf_as_na=inf_as_na) elif needs_i8_conversion(dtype): # this is the NaT pattern result = values.view("i8") == iNaT else: - if old: + if inf_as_na: result = ~np.isfinite(values) else: result = np.isnan(values) @@ -231,7 +231,9 @@ def _isna_ndarraylike(obj, old: bool = False): return result -def _isna_string_dtype(values: np.ndarray, dtype: np.dtype, old: bool) -> np.ndarray: +def _isna_string_dtype( + values: np.ndarray, dtype: np.dtype, inf_as_na: bool +) -> np.ndarray: # Working around NumPy ticket 1542 shape = values.shape @@ -239,7 +241,7 @@ def _isna_string_dtype(values: np.ndarray, dtype: np.dtype, old: bool) -> np.nda result = np.zeros(values.shape, dtype=bool) else: result = np.empty(shape, dtype=bool) - if old: + if inf_as_na: vec = libmissing.isnaobj_old(values.ravel()) else: vec = libmissing.isnaobj(values.ravel()) From db34f48856de536f2831af30c30c693964c39430 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Tue, 28 Apr 2020 11:51:04 -0500 Subject: [PATCH 3/3] Fix docstring --- pandas/core/dtypes/missing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index bdc9227909d90..87ff9a78909dd 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -128,12 +128,12 @@ def isna(obj): def _isna(obj, inf_as_na: bool = False): """ - Detect missing values, treating None, NaN or NA as null. If old is True - infinite values will also be treated as null. + Detect missing values, treating None, NaN or NA as null. Infinite + values will also be treated as null if inf_as_na is True. Parameters ---------- - arr: ndarray or object value + obj: ndarray or object value Input array or scalar value. inf_as_na: bool Whether to treat infinity as null.