From 0e60b5c077d0edde7de4e508d143f30c360fd876 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 8 Jul 2019 14:26:24 -0700 Subject: [PATCH 1/2] Patch can_hold_element TimedeltaBlock and DatetimeBlock Confirm that the try/except behavior for fillna is equivalent to checking _can_hold_element --- pandas/core/internals/blocks.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index bf6ebf1abe760..58f24e587cbc4 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -7,7 +7,7 @@ import numpy as np -from pandas._libs import lib, tslib, tslibs +from pandas._libs import NaT, lib, tslib, tslibs import pandas._libs.internals as libinternals from pandas._libs.tslibs import Timedelta, conversion, is_null_datetimelike from pandas.util._validators import validate_bool_kwarg @@ -413,7 +413,9 @@ def fillna(self, value, limit=None, inplace=False, downcast=None): try: # Note: we only call try_coerce_args to let it raise self._try_coerce_args(value) + assert self._can_hold_element(value), value except (TypeError, ValueError): + assert not self._can_hold_element(value), value # we can't process the value, but nothing to do if not mask.any(): @@ -2275,7 +2277,13 @@ def _can_hold_element(self, element): tipo = maybe_infer_dtype_type(element) if tipo is not None: return tipo == _NS_DTYPE or tipo == np.int64 - return is_integer(element) or isinstance(element, datetime) or isna(element) + if isinstance(element, datetime): + return element.tzinfo is None + if is_integer(element): + return element == tslibs.iNaT + + # TODO: shouldnt we exclude timedelta64("NaT")? See GH#27297 + return isna(element) def _coerce_values(self, values): return values.view("i8") @@ -2627,6 +2635,8 @@ def _can_hold_element(self, element): tipo = maybe_infer_dtype_type(element) if tipo is not None: return issubclass(tipo.type, (np.timedelta64, np.int64)) + if element is NaT: + return True return is_integer(element) or isinstance( element, (timedelta, np.timedelta64, np.int64) ) From eb6eaae68da7d4c9d8cf7cb6eb645d6648c9a22a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 8 Jul 2019 14:45:53 -0700 Subject: [PATCH 2/2] Change try/except to can_hold_element check --- pandas/core/internals/blocks.py | 40 ++++++++++++++------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 58f24e587cbc4..be0661c895485 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -409,35 +409,29 @@ def fillna(self, value, limit=None, inplace=False, downcast=None): else: return self.copy() - # fillna, but if we cannot coerce, then try again as an ObjectBlock - try: - # Note: we only call try_coerce_args to let it raise - self._try_coerce_args(value) - assert self._can_hold_element(value), value - except (TypeError, ValueError): - assert not self._can_hold_element(value), value - - # we can't process the value, but nothing to do - if not mask.any(): - return self if inplace else self.copy() - - # operate column-by-column - def f(m, v, i): - block = self.coerce_to_target_dtype(value) - - # slice out our block - if i is not None: - block = block.getitem_block(slice(i, i + 1)) - return block.fillna(value, limit=limit, inplace=inplace, downcast=None) - - return self.split_and_operate(mask, f, inplace) - else: + if self._can_hold_element(value): + # equivalent: self._try_coerce_args(value) would not raise blocks = self.putmask(mask, value, inplace=inplace) blocks = [ b.make_block(values=self._try_coerce_result(b.values)) for b in blocks ] return self._maybe_downcast(blocks, downcast) + # we can't process the value, but nothing to do + if not mask.any(): + return self if inplace else self.copy() + + # operate column-by-column + def f(m, v, i): + block = self.coerce_to_target_dtype(value) + + # slice out our block + if i is not None: + block = block.getitem_block(slice(i, i + 1)) + return block.fillna(value, limit=limit, inplace=inplace, downcast=None) + + return self.split_and_operate(mask, f, inplace) + def split_and_operate(self, mask, f, inplace): """ split the block per-column, and apply the callable f