From f68b2bf995ce2fa31bae10c163694b1bc63a0057 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 20 Feb 2021 09:57:45 -0800 Subject: [PATCH 1/2] REF: share _maybe_coerce_values --- pandas/core/internals/blocks.py | 68 ++++++++++----------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 1377928f71915..d341ce4906ab1 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -17,7 +17,6 @@ from pandas._libs import ( Interval, - NaT, Period, Timestamp, algos as libalgos, @@ -44,6 +43,7 @@ maybe_downcast_numeric, maybe_downcast_to_dtype, maybe_upcast, + sanitize_to_nanoseconds, soft_convert_objects, ) from pandas.core.dtypes.common import ( @@ -71,6 +71,7 @@ from pandas.core.dtypes.missing import ( is_valid_na_for_dtype, isna, + na_value_for_dtype, ) import pandas.core.algorithms as algos @@ -98,7 +99,10 @@ ) from pandas.core.base import PandasObject import pandas.core.common as com -from pandas.core.construction import extract_array +from pandas.core.construction import ( + ensure_wrapped_if_datetimelike, + extract_array, +) from pandas.core.indexers import ( check_setitem_lengths, is_empty_indexer, @@ -2081,7 +2085,6 @@ def fillna( class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock): """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock.""" - _dtype: np.dtype _holder: Type[Union[DatetimeArray, TimedeltaArray]] @classmethod @@ -2098,25 +2101,26 @@ def _maybe_coerce_values(cls, values): Returns ------- values : ndarray[datetime64ns/timedelta64ns] - - Overridden by DatetimeTZBlock. """ - if values.dtype != cls._dtype: - # non-nano we will convert to nano - if values.dtype.kind != cls._dtype.kind: - # caller is responsible for ensuring td64/dt64 dtype - raise TypeError(values.dtype) # pragma: no cover - - values = cls._holder._from_sequence(values)._data - - if isinstance(values, cls._holder): + values = extract_array(values, extract_numpy=True) + if isinstance(values, np.ndarray): + values = sanitize_to_nanoseconds(values) + elif isinstance(values.dtype, np.dtype): + # i.e. not datetime64tz values = values._data - assert isinstance(values, np.ndarray), type(values) return values def array_values(self): - return self._holder._simple_new(self.values) + return ensure_wrapped_if_datetimelike(self.values) + + @property + def _holder(self): + return type(self.array_values()) + + @property + def fill_value(self): + return na_value_for_dtype(self.dtype) def to_native_types(self, na_rep="NaT", **kwargs): """ convert to our native types format """ @@ -2129,9 +2133,6 @@ def to_native_types(self, na_rep="NaT", **kwargs): class DatetimeBlock(DatetimeLikeBlockMixin): __slots__ = () is_datetime = True - fill_value = np.datetime64("NaT", "ns") - _dtype = fill_value.dtype - _holder = DatetimeArray @property def _can_hold_na(self): @@ -2154,42 +2155,16 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock): __slots__ = () is_extension = True - _holder = DatetimeArray - internal_values = Block.internal_values _can_hold_element = DatetimeBlock._can_hold_element to_native_types = DatetimeBlock.to_native_types diff = DatetimeBlock.diff - fill_value = NaT where = DatetimeBlock.where putmask = DatetimeLikeBlockMixin.putmask fillna = DatetimeLikeBlockMixin.fillna array_values = ExtensionBlock.array_values - @classmethod - def _maybe_coerce_values(cls, values): - """ - Input validation for values passed to __init__. Ensure that - we have datetime64TZ, coercing if necessary. - - Parameters - ---------- - values : array-like - Must be convertible to datetime64 - - Returns - ------- - values : DatetimeArray - """ - if not isinstance(values, cls._holder): - values = cls._holder(values) - - if values.tz is None: - raise ValueError("cannot create a DatetimeTZBlock without a tz") - - return values - @property def is_view(self) -> bool: """ return a boolean if I am possibly a view """ @@ -2207,9 +2182,6 @@ class TimeDeltaBlock(DatetimeLikeBlockMixin): __slots__ = () _can_hold_na = True is_numeric = False - _holder = TimedeltaArray - fill_value = np.timedelta64("NaT", "ns") - _dtype = fill_value.dtype class ObjectBlock(Block): From 44cc274a4135c46f2c46924bfef92f7a7fda1f63 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 20 Feb 2021 11:22:49 -0800 Subject: [PATCH 2/2] mypy fixup --- pandas/core/internals/blocks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index d341ce4906ab1..ef6b7e524a962 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -95,7 +95,6 @@ DatetimeArray, ExtensionArray, PandasArray, - TimedeltaArray, ) from pandas.core.base import PandasObject import pandas.core.common as com @@ -2085,8 +2084,6 @@ def fillna( class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock): """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock.""" - _holder: Type[Union[DatetimeArray, TimedeltaArray]] - @classmethod def _maybe_coerce_values(cls, values): """