From 7cb6295cafe5ad16d0dc1136efa9572233b09716 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:16:32 -0800 Subject: [PATCH 01/33] remove ensure_datetimelike_to_i8 --- pandas/core/arrays/datetimelike.py | 56 +++++++---------------------- pandas/core/indexes/datetimelike.py | 21 ++++++----- 2 files changed, 25 insertions(+), 52 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 0fadf3a05a1d3..f3e62eb6586b6 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -34,7 +34,7 @@ is_unsigned_integer_dtype, pandas_dtype, ) -from pandas.core.dtypes.generic import ABCIndexClass, ABCPeriodArray, ABCSeries +from pandas.core.dtypes.generic import ABCSeries from pandas.core.dtypes.inference import is_array_like from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna @@ -368,16 +368,19 @@ class TimelikeOps: def _round(self, freq, mode, ambiguous, nonexistent): # round the local times - values = _ensure_datetimelike_to_i8(self) + if is_datetime64tz_dtype(self): + # operate on naive timestamps, then convert back to aware + naive = self.tz_localize(None) + result = naive._round(freq, mode, ambiguous, nonexistent) + aware = result.tz_localize( + self.tz, mode=mode, ambiguous=ambiguous, nonexistent=nonexistent + ) + return aware + + values = self.view("i8") result = round_nsint64(values, mode, freq) result = self._maybe_mask_results(result, fill_value=NaT) - - dtype = self.dtype - if is_datetime64tz_dtype(self): - dtype = None - return self._ensure_localized( - self._simple_new(result, dtype=dtype), ambiguous, nonexistent - ) + return self._simple_new(result, dtype=self.dtype) @Appender((_round_doc + _round_example).format(op="round")) def round(self, freq, ambiguous="raise", nonexistent="raise"): @@ -1687,38 +1690,3 @@ def maybe_infer_freq(freq): freq_infer = True freq = None return freq, freq_infer - - -def _ensure_datetimelike_to_i8(other, to_utc=False): - """ - Helper for coercing an input scalar or array to i8. - - Parameters - ---------- - other : 1d array - to_utc : bool, default False - If True, convert the values to UTC before extracting the i8 values - If False, extract the i8 values directly. - - Returns - ------- - i8 1d array - """ - from pandas import Index - - if lib.is_scalar(other) and isna(other): - return iNaT - elif isinstance(other, (ABCPeriodArray, ABCIndexClass, DatetimeLikeArrayMixin)): - # convert tz if needed - if getattr(other, "tz", None) is not None: - if to_utc: - other = other.tz_convert("UTC") - else: - other = other.tz_localize(None) - else: - try: - return np.array(other, copy=False).view("i8") - except TypeError: - # period array cannot be coerced to int - other = Index(other) - return other.asi8 diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 3a58794a8b19e..104b3680e31fb 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -22,6 +22,7 @@ is_list_like, is_period_dtype, is_scalar, + needs_i8_conversion, ) from pandas.core.dtypes.concat import concat_compat from pandas.core.dtypes.generic import ABCIndex, ABCIndexClass, ABCSeries @@ -34,10 +35,7 @@ ExtensionOpsMixin, TimedeltaArray, ) -from pandas.core.arrays.datetimelike import ( - DatetimeLikeArrayMixin, - _ensure_datetimelike_to_i8, -) +from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs from pandas.core.indexes.numeric import Int64Index @@ -491,11 +489,18 @@ def repeat(self, repeats, axis=None): @Appender(_index_shared_docs["where"] % _index_doc_kwargs) def where(self, cond, other=None): - other = _ensure_datetimelike_to_i8(other, to_utc=True) - values = _ensure_datetimelike_to_i8(self, to_utc=True) - result = np.where(cond, values, other).astype("i8") + values = self.view("i8") + if not needs_i8_conversion(other): + # Primarily we want self.dtype, but could also be Categorical + # holding self.dtype + odtype = getattr(other, "dtype", None) + raise TypeError(f"Where requires matching dtype, not {odtype}") - result = self._ensure_localized(result, from_utc=True) + other = type(self._data)._from_sequence(other) + # TODO: require dtype match + other = other.view("i8") + + result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) def _summary(self, name=None): From 6d33f16286da2df166b7887f35c55b026c230366 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:30:02 -0800 Subject: [PATCH 02/33] typo fixup --- pandas/core/arrays/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index f3e62eb6586b6..8b4a3720d9a73 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -373,7 +373,7 @@ def _round(self, freq, mode, ambiguous, nonexistent): naive = self.tz_localize(None) result = naive._round(freq, mode, ambiguous, nonexistent) aware = result.tz_localize( - self.tz, mode=mode, ambiguous=ambiguous, nonexistent=nonexistent + self.tz, ambiguous=ambiguous, nonexistent=nonexistent ) return aware From 6bc3cdbd1f49200839e19f4ee86a93028739d6cc Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:34:54 -0800 Subject: [PATCH 03/33] handle categories --- pandas/core/indexes/datetimelike.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 104b3680e31fb..1b39e1726ab5a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -16,6 +16,7 @@ from pandas.core.dtypes.common import ( ensure_int64, is_bool_dtype, + is_categorical_dtype, is_dtype_equal, is_float, is_integer, @@ -490,6 +491,10 @@ def repeat(self, repeats, axis=None): @Appender(_index_shared_docs["where"] % _index_doc_kwargs) def where(self, cond, other=None): values = self.view("i8") + + if is_categorical_dtype(other): + if needs_i8_conversion(other.categories): + other._internal_get_values() if not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype From 3d69e343c0dd212c775779f9749a5acca4a13835 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:35:29 -0800 Subject: [PATCH 04/33] handle categories --- pandas/core/indexes/datetimelike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 1b39e1726ab5a..b0703e846369e 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -495,6 +495,8 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): other._internal_get_values() + else: + raise TypeError(other.dtype) if not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype From c6d85587339f7e3ba624278dfd3fd552e01bc564 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:36:00 -0800 Subject: [PATCH 05/33] handle categories --- pandas/core/indexes/datetimelike.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index b0703e846369e..077552aa03f99 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -495,6 +495,7 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): other._internal_get_values() + raise TypeError(other.dtype) else: raise TypeError(other.dtype) if not needs_i8_conversion(other): From c6c8ae53172db60f3507c255d27b7d500a3b5120 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:36:51 -0800 Subject: [PATCH 06/33] handle categories --- pandas/core/indexes/datetimelike.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 077552aa03f99..14e62bab655fd 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -495,6 +495,7 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): other._internal_get_values() + assert needs_i8_conversion(other), other.dtype raise TypeError(other.dtype) else: raise TypeError(other.dtype) From 975e4aa3f52f4dd4ccf852fa60fbc938cdc0f840 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:37:05 -0800 Subject: [PATCH 07/33] handle categories --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 14e62bab655fd..18e6c49c6cf1a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -495,7 +495,7 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): other._internal_get_values() - assert needs_i8_conversion(other), other.dtype + assert needs_i8_conversion(other), other.categories.dtype raise TypeError(other.dtype) else: raise TypeError(other.dtype) From 546872af1aef51bd63fe104a8734e5889a07103d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:37:58 -0800 Subject: [PATCH 08/33] handle categories --- pandas/core/indexes/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 18e6c49c6cf1a..e0f17d3b56640 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -494,9 +494,9 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): - other._internal_get_values() + getattr(other, "_data", other)._internal_get_values() assert needs_i8_conversion(other), other.categories.dtype - raise TypeError(other.dtype) + #raise TypeError(other.dtype) else: raise TypeError(other.dtype) if not needs_i8_conversion(other): From 459880125eb09dc0b77099d4c33a39a38d91c8f9 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:38:29 -0800 Subject: [PATCH 09/33] handle categories --- pandas/core/indexes/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index e0f17d3b56640..818c162472e9d 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -494,8 +494,8 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): - getattr(other, "_data", other)._internal_get_values() - assert needs_i8_conversion(other), other.categories.dtype + other = other._internal_get_values() + assert needs_i8_conversion #raise TypeError(other.dtype) else: raise TypeError(other.dtype) From 4b1572352ff2bb074416f08d1e5dc01b42522a41 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:38:51 -0800 Subject: [PATCH 10/33] handle categories --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 818c162472e9d..5dd41fa79620a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -495,7 +495,7 @@ def where(self, cond, other=None): if is_categorical_dtype(other): if needs_i8_conversion(other.categories): other = other._internal_get_values() - assert needs_i8_conversion + assert needs_i8_conversion(other) #raise TypeError(other.dtype) else: raise TypeError(other.dtype) From a4d338ca856e00fb14641e6157237fbf0b6b6004 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:39:09 -0800 Subject: [PATCH 11/33] handle categories --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 5dd41fa79620a..d2dff0c3eff9a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -503,7 +503,7 @@ def where(self, cond, other=None): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype odtype = getattr(other, "dtype", None) - raise TypeError(f"Where requires matching dtype, not {odtype}") + raise TypeError(f"Where requires matching dtype, not {odtype}", type(other)) other = type(self._data)._from_sequence(other) # TODO: require dtype match From fad9b7e2ef328edda13860f75c016f3711484f88 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:40:45 -0800 Subject: [PATCH 12/33] handle None --- pandas/core/indexes/datetimelike.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index d2dff0c3eff9a..7ea2f7e2a9226 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -499,15 +499,16 @@ def where(self, cond, other=None): #raise TypeError(other.dtype) else: raise TypeError(other.dtype) - if not needs_i8_conversion(other): + if not is_scalar(other) and not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype odtype = getattr(other, "dtype", None) raise TypeError(f"Where requires matching dtype, not {odtype}", type(other)) - other = type(self._data)._from_sequence(other) - # TODO: require dtype match - other = other.view("i8") + if not is_scalar(other): + other = type(self._data)._from_sequence(other) + # TODO: require dtype match + other = other.view("i8") result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) From 466679ece428ab525c974e01e4d47a9a2f191fd0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:41:21 -0800 Subject: [PATCH 13/33] handle None --- pandas/core/indexes/datetimelike.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 7ea2f7e2a9226..3e0ece3be6c0b 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -509,6 +509,10 @@ def where(self, cond, other=None): other = type(self._data)._from_sequence(other) # TODO: require dtype match other = other.view("i8") + elif other is None: + if cond.all(): + # Then it doesnt matter what other is, so go with it + other = NaT.value result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) From 727afc6d8a1e6911ae30aaafb8029fa5fdce1143 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:41:53 -0800 Subject: [PATCH 14/33] handle None --- pandas/core/indexes/datetimelike.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 3e0ece3be6c0b..8dd0ae60a6212 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -513,6 +513,10 @@ def where(self, cond, other=None): if cond.all(): # Then it doesnt matter what other is, so go with it other = NaT.value + else: + raise TypeError(other) + else: + raise TypeError(other) result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) From 31ed9b804488b82e7f59be1f9be25037ff72cc9e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:42:26 -0800 Subject: [PATCH 15/33] handle None --- pandas/core/indexes/datetimelike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 8dd0ae60a6212..8c28f44c14d61 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -514,7 +514,8 @@ def where(self, cond, other=None): # Then it doesnt matter what other is, so go with it other = NaT.value else: - raise TypeError(other) + other = NaT.value + #raise TypeError(other) else: raise TypeError(other) From eaa34b0529100a68c1bd520f33978324e73f5bd7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:44:46 -0800 Subject: [PATCH 16/33] handle None --- pandas/core/indexes/datetimelike.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 8c28f44c14d61..4fc8ab2e2ecc9 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -510,10 +510,13 @@ def where(self, cond, other=None): # TODO: require dtype match other = other.view("i8") elif other is None: - if cond.all(): + if not hasattr(cond, "all"): + other = NaT.value + elif cond.all(): # Then it doesnt matter what other is, so go with it other = NaT.value else: + # 7 tests other = NaT.value #raise TypeError(other) else: From ba0e555d0d4af6a4eca22b85405f2cc991844e83 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:45:14 -0800 Subject: [PATCH 17/33] handle None --- pandas/core/indexes/datetimelike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 4fc8ab2e2ecc9..fda9280da75b8 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -519,6 +519,8 @@ def where(self, cond, other=None): # 7 tests other = NaT.value #raise TypeError(other) + elif is_scalar(other) and np.isnan(other): + other = NaT.value else: raise TypeError(other) From 24f9327ddbafa1ecb60313bd5ac90f009616466a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:46:50 -0800 Subject: [PATCH 18/33] handle None --- pandas/core/indexes/datetimelike.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index fda9280da75b8..cd71953d09146 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -499,6 +499,11 @@ def where(self, cond, other=None): #raise TypeError(other.dtype) else: raise TypeError(other.dtype) + if is_object_dtype(other): + # Try to do inference, e.g. we passed PeriodIndex.values and got + # an ndarray of Periods + other = pd.Index(np.asarray(other)) + if not is_scalar(other) and not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype From 17fe5bb0935d94841bd916b1836b77d258d549f3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:47:05 -0800 Subject: [PATCH 19/33] handle None --- pandas/core/indexes/datetimelike.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index cd71953d09146..b8f9584f6b39d 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -19,6 +19,7 @@ is_categorical_dtype, is_dtype_equal, is_float, + is_object_dtype, is_integer, is_list_like, is_period_dtype, From 39a398ca358d8dfabd10baa2c01647442b728566 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:47:22 -0800 Subject: [PATCH 20/33] handle None --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index b8f9584f6b39d..6824b414438da 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -503,7 +503,7 @@ def where(self, cond, other=None): if is_object_dtype(other): # Try to do inference, e.g. we passed PeriodIndex.values and got # an ndarray of Periods - other = pd.Index(np.asarray(other)) + other = Index(np.asarray(other)) if not is_scalar(other) and not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical From 679aa96aab0c70586f68752246c4eb1f203cfbbe Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:58:02 -0800 Subject: [PATCH 21/33] handle None --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 6824b414438da..198c1b1b76821 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -525,7 +525,7 @@ def where(self, cond, other=None): # 7 tests other = NaT.value #raise TypeError(other) - elif is_scalar(other) and np.isnan(other): + elif is_float(other) and np.isnan(other): other = NaT.value else: raise TypeError(other) From d0a7670492b4fddc8c1c7b5d3bada1b230d67a8e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 09:58:53 -0800 Subject: [PATCH 22/33] handle None --- pandas/core/indexes/datetimelike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 198c1b1b76821..cb8d2ea8adc97 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -528,6 +528,8 @@ def where(self, cond, other=None): elif is_float(other) and np.isnan(other): other = NaT.value else: + # We test for error message coming from the Index constructor + Index(other) raise TypeError(other) result = np.where(cond, values, other).astype("i8") From f62cb902159b7b4edf3c587ebb914b14f63dc250 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:11:09 -0800 Subject: [PATCH 23/33] simplify --- pandas/core/indexes/datetimelike.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index cb8d2ea8adc97..a2befd103f7da 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -493,17 +493,14 @@ def repeat(self, repeats, axis=None): def where(self, cond, other=None): values = self.view("i8") + # Do type inference if necessary up front + # e.g. we passed PeriodIndex.values and got an ndarray of Periods + other = Index(other) + if is_categorical_dtype(other): + # e.g. we have a Categorical holding self.dtype if needs_i8_conversion(other.categories): other = other._internal_get_values() - assert needs_i8_conversion(other) - #raise TypeError(other.dtype) - else: - raise TypeError(other.dtype) - if is_object_dtype(other): - # Try to do inference, e.g. we passed PeriodIndex.values and got - # an ndarray of Periods - other = Index(np.asarray(other)) if not is_scalar(other) and not needs_i8_conversion(other): # Primarily we want self.dtype, but could also be Categorical From bbb0568d9d2ccdad6d3180e193c531e94d9b3535 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:16:08 -0800 Subject: [PATCH 24/33] simplify --- pandas/core/indexes/datetimelike.py | 42 ++++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index a2befd103f7da..0ff28f78f3a47 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -493,9 +493,13 @@ def repeat(self, repeats, axis=None): def where(self, cond, other=None): values = self.view("i8") - # Do type inference if necessary up front - # e.g. we passed PeriodIndex.values and got an ndarray of Periods - other = Index(other) + if is_scalar(other) and isna(other): + other = NaT.value + + else: + # Do type inference if necessary up front + # e.g. we passed PeriodIndex.values and got an ndarray of Periods + other = Index(other) if is_categorical_dtype(other): # e.g. we have a Categorical holding self.dtype @@ -512,22 +516,22 @@ def where(self, cond, other=None): other = type(self._data)._from_sequence(other) # TODO: require dtype match other = other.view("i8") - elif other is None: - if not hasattr(cond, "all"): - other = NaT.value - elif cond.all(): - # Then it doesnt matter what other is, so go with it - other = NaT.value - else: - # 7 tests - other = NaT.value - #raise TypeError(other) - elif is_float(other) and np.isnan(other): - other = NaT.value - else: - # We test for error message coming from the Index constructor - Index(other) - raise TypeError(other) + #elif other is None: + # if not hasattr(cond, "all"): + # other = NaT.value + # elif cond.all(): + # # Then it doesnt matter what other is, so go with it + # other = NaT.value + # else: + # # 7 tests + # other = NaT.value + # #raise TypeError(other) + #elif is_float(other) and np.isnan(other): + # other = NaT.value + #else: + # # We test for error message coming from the Index constructor + # Index(other) + # raise TypeError(other) result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) From 08eabab9807ac56eebe1f6f43b59489c69a32509 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:16:35 -0800 Subject: [PATCH 25/33] simplify --- pandas/core/indexes/datetimelike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 0ff28f78f3a47..d15e497630f9f 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -19,15 +19,16 @@ is_categorical_dtype, is_dtype_equal, is_float, - is_object_dtype, is_integer, is_list_like, + is_object_dtype, is_period_dtype, is_scalar, needs_i8_conversion, ) from pandas.core.dtypes.concat import concat_compat from pandas.core.dtypes.generic import ABCIndex, ABCIndexClass, ABCSeries +from pandas.core.dtypes.missing import isna from pandas.core import algorithms from pandas.core.accessor import PandasDelegate From db99643fb8b92a948d46d167667974acf09eac16 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:22:34 -0800 Subject: [PATCH 26/33] simplify --- pandas/core/indexes/datetimelike.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index d15e497630f9f..0820d2f3a57be 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -21,7 +21,6 @@ is_float, is_integer, is_list_like, - is_object_dtype, is_period_dtype, is_scalar, needs_i8_conversion, @@ -507,32 +506,15 @@ def where(self, cond, other=None): if needs_i8_conversion(other.categories): other = other._internal_get_values() - if not is_scalar(other) and not needs_i8_conversion(other): + if not is_scalar(other) and not is_dtype_equal(self.dtype, other.dtype): # Primarily we want self.dtype, but could also be Categorical # holding self.dtype - odtype = getattr(other, "dtype", None) - raise TypeError(f"Where requires matching dtype, not {odtype}", type(other)) + raise TypeError(f"Where requires matching dtype, not {other.dtype}") if not is_scalar(other): - other = type(self._data)._from_sequence(other) + #other = type(self._data)._from_sequence(other) # TODO: require dtype match other = other.view("i8") - #elif other is None: - # if not hasattr(cond, "all"): - # other = NaT.value - # elif cond.all(): - # # Then it doesnt matter what other is, so go with it - # other = NaT.value - # else: - # # 7 tests - # other = NaT.value - # #raise TypeError(other) - #elif is_float(other) and np.isnan(other): - # other = NaT.value - #else: - # # We test for error message coming from the Index constructor - # Index(other) - # raise TypeError(other) result = np.where(cond, values, other).astype("i8") return self._shallow_copy(result) From 6d7a429c1babf1d98d5018ca53d286af45ee8d7b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:28:05 -0800 Subject: [PATCH 27/33] fix test --- pandas/tests/indexes/datetimes/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 210b28aa0c393..62a8d39b6dfd5 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -132,7 +132,7 @@ def test_where_other(self): i2 = i.copy() i2 = Index([pd.NaT, pd.NaT] + i[2:].tolist()) - result = i.where(notna(i2), i2.values) + result = i.where(notna(i2), i2._values) tm.assert_index_equal(result, i2) def test_where_tz(self): From ee6db8a630186b512d57575263318e549bff0bb9 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:30:07 -0800 Subject: [PATCH 28/33] extend test --- pandas/tests/indexes/datetimes/test_indexing.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 62a8d39b6dfd5..2771337aba1d3 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -135,6 +135,20 @@ def test_where_other(self): result = i.where(notna(i2), i2._values) tm.assert_index_equal(result, i2) + with pytest.raises(TypeError, match="Where requires matching dtype"): + # passing tz-naive ndarray to tzaware DTI + i.where(notna(i2), i2.values) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + # passing tz-aware DTI to tznaive DTI + i.tz_localize(None).where(notna(i2), i2) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + i.where(notna(i2), i2.to_period("D")) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + i.where(notna(i2), i2.asi8.view("timedelta64[ns]") + def test_where_tz(self): i = pd.date_range("20130101", periods=3, tz="US/Eastern") result = i.where(notna(i)) From 730cbd159a77c0614effa1a72309b6e8a46febc6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:30:40 -0800 Subject: [PATCH 29/33] typo fixup --- pandas/tests/indexes/datetimes/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 2771337aba1d3..4185f336365bf 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -147,7 +147,7 @@ def test_where_other(self): i.where(notna(i2), i2.to_period("D")) with pytest.raises(TypeError, match="Where requires matching dtype"): - i.where(notna(i2), i2.asi8.view("timedelta64[ns]") + i.where(notna(i2), i2.asi8.view("timedelta64[ns]")) def test_where_tz(self): i = pd.date_range("20130101", periods=3, tz="US/Eastern") From 2448aa43b39cbd2366fe4cd68632d3a95ca54a02 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:44:40 -0800 Subject: [PATCH 30/33] tests --- .../tests/indexes/datetimes/test_indexing.py | 17 +++++++++++++---- pandas/tests/indexes/period/test_indexing.py | 15 +++++++++++++++ .../tests/indexes/timedeltas/test_indexing.py | 19 ++++++++++++++++--- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 4185f336365bf..97290c8c635b8 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -135,19 +135,28 @@ def test_where_other(self): result = i.where(notna(i2), i2._values) tm.assert_index_equal(result, i2) + def test_where_invalid_dtypes(self): + dti = pd.date_range("20130101", periods=3, tz="US/Eastern") + + i2 = dti.copy() + i2 = Index([pd.NaT, pd.NaT] + dti[2:].tolist()) + with pytest.raises(TypeError, match="Where requires matching dtype"): # passing tz-naive ndarray to tzaware DTI - i.where(notna(i2), i2.values) + dti.where(notna(i2), i2.values) with pytest.raises(TypeError, match="Where requires matching dtype"): # passing tz-aware DTI to tznaive DTI - i.tz_localize(None).where(notna(i2), i2) + dti.tz_localize(None).where(notna(i2), i2) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + dti.where(notna(i2), i2.tz_localize(None).to_period("D")) with pytest.raises(TypeError, match="Where requires matching dtype"): - i.where(notna(i2), i2.to_period("D")) + dti.where(notna(i2), i2.asi8.view("timedelta64[ns]")) with pytest.raises(TypeError, match="Where requires matching dtype"): - i.where(notna(i2), i2.asi8.view("timedelta64[ns]")) + dti.where(notna(i2), i2.asi8) def test_where_tz(self): i = pd.date_range("20130101", periods=3, tz="US/Eastern") diff --git a/pandas/tests/indexes/period/test_indexing.py b/pandas/tests/indexes/period/test_indexing.py index e95b4ae5397b4..7dbefbdaff98e 100644 --- a/pandas/tests/indexes/period/test_indexing.py +++ b/pandas/tests/indexes/period/test_indexing.py @@ -235,6 +235,21 @@ def test_where_other(self): result = i.where(notna(i2), i2.values) tm.assert_index_equal(result, i2) + def test_where_invalid_dtypes(self): + pi = period_range("20130101", periods=5, freq="D") + + i2 = pi.copy() + i2 = pd.PeriodIndex([pd.NaT, pd.NaT] + pi[2:].tolist(), freq="D") + + with pytest.raises(TypeError, match="Where requires matching dtype"): + pi.where(notna(i2), i2.asi8) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + pi.where(notna(i2), i2.asi8.view("timedelta64[ns]")) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + pi.where(notna(i2), i2.to_timestamp("S")) + class TestTake: def test_take(self): diff --git a/pandas/tests/indexes/timedeltas/test_indexing.py b/pandas/tests/indexes/timedeltas/test_indexing.py index b70a3d17a10ab..cbd022961693b 100644 --- a/pandas/tests/indexes/timedeltas/test_indexing.py +++ b/pandas/tests/indexes/timedeltas/test_indexing.py @@ -4,7 +4,7 @@ import pytest import pandas as pd -from pandas import Index, Timedelta, TimedeltaIndex, timedelta_range +from pandas import Index, Timedelta, TimedeltaIndex, timedelta_range, notna import pandas._testing as tm @@ -58,8 +58,21 @@ def test_timestamp_invalid_key(self, key): class TestWhere: - # placeholder for symmetry with DatetimeIndex and PeriodIndex tests - pass + + def test_where_invalid_dtypes(self): + tdi = timedelta_range("1 day", periods=3, freq="D", name="idx") + + i2 = tdi.copy() + i2 = Index([pd.NaT, pd.NaT] + tdi[2:].tolist()) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + tdi.where(notna(i2), i2.asi8) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + tdi.where(notna(i2), i2 + pd.Timestamp.now()) + + with pytest.raises(TypeError, match="Where requires matching dtype"): + tdi.where(notna(i2), (i2 + pd.Timestamp.now()).to_period("D")) class TestTake: From 00d7415a8200f69185604c964bb29bf0724c3efb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:49:16 -0800 Subject: [PATCH 31/33] cleanup --- pandas/core/indexes/datetimelike.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 0820d2f3a57be..19733c7cd6e69 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -501,19 +501,14 @@ def where(self, cond, other=None): # e.g. we passed PeriodIndex.values and got an ndarray of Periods other = Index(other) - if is_categorical_dtype(other): - # e.g. we have a Categorical holding self.dtype - if needs_i8_conversion(other.categories): - other = other._internal_get_values() - - if not is_scalar(other) and not is_dtype_equal(self.dtype, other.dtype): - # Primarily we want self.dtype, but could also be Categorical - # holding self.dtype - raise TypeError(f"Where requires matching dtype, not {other.dtype}") - - if not is_scalar(other): - #other = type(self._data)._from_sequence(other) - # TODO: require dtype match + if is_categorical_dtype(other): + # e.g. we have a Categorical holding self.dtype + if needs_i8_conversion(other.categories): + other = other._internal_get_values() + + if not is_dtype_equal(self.dtype, other.dtype): + raise TypeError(f"Where requires matching dtype, not {other.dtype}") + other = other.view("i8") result = np.where(cond, values, other).astype("i8") From 364765d89b0495cce85b5826f74425f206939e8f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 10:50:58 -0800 Subject: [PATCH 32/33] remove _ensure_localize --- pandas/core/arrays/datetimelike.py | 39 ----------------------------- pandas/core/indexes/datetimelike.py | 12 --------- 2 files changed, 51 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 8b4a3720d9a73..4a37b4f0f29ba 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1414,45 +1414,6 @@ def __isub__(self, other): # type: ignore self._freq = result._freq return self - # -------------------------------------------------------------- - # Comparison Methods - - def _ensure_localized( - self, arg, ambiguous="raise", nonexistent="raise", from_utc=False - ): - """ - Ensure that we are re-localized. - - This is for compat as we can then call this on all datetimelike - arrays generally (ignored for Period/Timedelta) - - Parameters - ---------- - arg : Union[DatetimeLikeArray, DatetimeIndexOpsMixin, ndarray] - ambiguous : str, bool, or bool-ndarray, default 'raise' - nonexistent : str, default 'raise' - from_utc : bool, default False - If True, localize the i8 ndarray to UTC first before converting to - the appropriate tz. If False, localize directly to the tz. - - Returns - ------- - localized array - """ - - # reconvert to local tz - tz = getattr(self, "tz", None) - if tz is not None: - if not isinstance(arg, type(self)): - arg = self._simple_new(arg) - if from_utc: - arg = arg.tz_localize("UTC").tz_convert(self.tz) - else: - arg = arg.tz_localize( - self.tz, ambiguous=ambiguous, nonexistent=nonexistent - ) - return arg - # -------------------------------------------------------------- # Reductions diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 19733c7cd6e69..bb8e04e87724f 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -177,18 +177,6 @@ def equals(self, other) -> bool: return np.array_equal(self.asi8, other.asi8) - def _ensure_localized( - self, arg, ambiguous="raise", nonexistent="raise", from_utc=False - ): - # See DatetimeLikeArrayMixin._ensure_localized.__doc__ - if getattr(self, "tz", None): - # ensure_localized is only relevant for tz-aware DTI - result = self._data._ensure_localized( - arg, ambiguous=ambiguous, nonexistent=nonexistent, from_utc=from_utc - ) - return type(self)._simple_new(result, name=self.name) - return arg - @Appender(_index_shared_docs["contains"] % _index_doc_kwargs) def __contains__(self, key): try: From 4b96ab256f5454e582ecf270511d6a2d8b1e9cb2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 7 Jan 2020 12:24:33 -0800 Subject: [PATCH 33/33] black/isort fixup --- pandas/tests/indexes/timedeltas/test_indexing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/indexes/timedeltas/test_indexing.py b/pandas/tests/indexes/timedeltas/test_indexing.py index cbd022961693b..36105477ba9ee 100644 --- a/pandas/tests/indexes/timedeltas/test_indexing.py +++ b/pandas/tests/indexes/timedeltas/test_indexing.py @@ -4,7 +4,7 @@ import pytest import pandas as pd -from pandas import Index, Timedelta, TimedeltaIndex, timedelta_range, notna +from pandas import Index, Timedelta, TimedeltaIndex, notna, timedelta_range import pandas._testing as tm @@ -58,7 +58,6 @@ def test_timestamp_invalid_key(self, key): class TestWhere: - def test_where_invalid_dtypes(self): tdi = timedelta_range("1 day", periods=3, freq="D", name="idx")