From 70c10de7a22d53b3be584f13b1d4948c4584c8c8 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 31 Jan 2020 10:47:56 -0600 Subject: [PATCH 1/7] BUG: Fixed IntervalArray[int].shift Closes https://github.com/pandas-dev/pandas/issues/31495 --- doc/source/whatsnew/v1.0.1.rst | 2 +- pandas/core/arrays/interval.py | 20 +++++++++++++++++++ pandas/tests/arrays/interval/test_interval.py | 8 ++++++++ pandas/tests/extension/base/methods.py | 7 +++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index ff8433c7cafd9..d21a7e3a3c62a 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -62,7 +62,7 @@ Strings Interval ^^^^^^^^ -- +- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers (:issue:`31495`) - Indexing diff --git a/pandas/core/arrays/interval.py b/pandas/core/arrays/interval.py index d890c0c16aecc..57b172439a88c 100644 --- a/pandas/core/arrays/interval.py +++ b/pandas/core/arrays/interval.py @@ -27,6 +27,7 @@ from pandas.core.dtypes.dtypes import IntervalDtype from pandas.core.dtypes.generic import ( ABCDatetimeIndex, + ABCExtensionArray, ABCIndexClass, ABCInterval, ABCIntervalIndex, @@ -788,6 +789,25 @@ def size(self) -> int: # Avoid materializing self.values return self.left.size + def shift(self, periods: int = 1, fill_value: object = None) -> ABCExtensionArray: + if not len(self) or periods == 0: + return self.copy() + + if isna(fill_value): + fill_value = self.dtype.na_value + + # Note: The only change from ExtensionArray.shift is the lack of + # dtype=self.dtype here, since the shifted dtype may not match the + # original dtype. + empty = self._from_sequence([fill_value] * min(abs(periods), len(self))) + if periods > 0: + a = empty + b = self[:-periods] + else: + a = self[abs(periods) :] + b = empty + return self._concat_same_type([a, b]) + def take(self, indices, allow_fill=False, fill_value=None, axis=None, **kwargs): """ Take elements from the IntervalArray. diff --git a/pandas/tests/arrays/interval/test_interval.py b/pandas/tests/arrays/interval/test_interval.py index 35eda4a0ec5bc..6e72a16fa9b00 100644 --- a/pandas/tests/arrays/interval/test_interval.py +++ b/pandas/tests/arrays/interval/test_interval.py @@ -81,6 +81,14 @@ def test_where_raises(self, other): with pytest.raises(ValueError, match=match): ser.where([True, False, True], other=other) + def test_shift(self): + # https://github.com/pandas-dev/pandas/issues/31495 + a = IntervalArray.from_breaks([1, 2, 3]) + result = a.shift() + # int -> float + expected = IntervalArray.from_tuples([(np.nan, np.nan), (1.0, 2.0)]) + tm.assert_interval_array_equal(result, expected) + class TestSetitem: def test_set_na(self, left_right_dtypes): diff --git a/pandas/tests/extension/base/methods.py b/pandas/tests/extension/base/methods.py index 4a84a21084de2..22e53dbc89f01 100644 --- a/pandas/tests/extension/base/methods.py +++ b/pandas/tests/extension/base/methods.py @@ -280,6 +280,13 @@ def test_shift_empty_array(self, data, periods): expected = empty self.assert_extension_array_equal(result, expected) + def test_shift_zero_copies(self, data): + result = data.shift(0) + assert result is not data + + result = data[:0].shift(2) + assert result is not data + def test_shift_fill_value(self, data): arr = data[:4] fill_value = data[0] From bafd770aacf82119f79a75d2a09c3ad285abbda6 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 3 Feb 2020 09:32:31 -0600 Subject: [PATCH 2/7] whatsnew --- doc/source/whatsnew/v1.0.1.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index 180411afb117d..7075deb3867cf 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -53,7 +53,11 @@ Bug fixes **Plotting** -- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`) +- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`** + +**Interval** + +- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers (:issue:`31495`) .. --------------------------------------------------------------------------- From 8d8bf88a7822629886d1abd684fbfb477c36fdae Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 3 Feb 2020 10:19:54 -0600 Subject: [PATCH 3/7] fixup --- doc/source/whatsnew/v1.0.1.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index 7075deb3867cf..4a92a420bd111 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -57,8 +57,7 @@ Bug fixes **Interval** -- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers (:issue:`31495`) - +- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers (:issue:`31495`) .. --------------------------------------------------------------------------- From 9c4a271de88279b85878b0fe9234f454855ce2e9 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 3 Feb 2020 10:25:38 -0600 Subject: [PATCH 4/7] Handle datetime64[ns] --- pandas/core/arrays/interval.py | 16 ++++++++++++---- pandas/tests/arrays/interval/test_interval.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pandas/core/arrays/interval.py b/pandas/core/arrays/interval.py index c49bd6cc32a22..0b35a031bc53f 100644 --- a/pandas/core/arrays/interval.py +++ b/pandas/core/arrays/interval.py @@ -797,10 +797,18 @@ def shift(self, periods: int = 1, fill_value: object = None) -> ABCExtensionArra if isna(fill_value): fill_value = self.dtype.na_value - # Note: The only change from ExtensionArray.shift is the lack of - # dtype=self.dtype here, since the shifted dtype may not match the - # original dtype. - empty = self._from_sequence([fill_value] * min(abs(periods), len(self))) + # ExtensionArray.shift doesn't work for two reasons + # 1. IntervalArray.dtype.na_value may not be correct for the dtype. + # 2. IntervalArray._from_sequence only accepts NaN for missing values, + # not other values like NaT + + empty_len = min(abs(periods), len(self)) + if isna(fill_value): + fill_value = self.left._na_value + empty = IntervalArray.from_breaks([fill_value] * (empty_len + 1)) + else: + empty = self._from_sequence([fill_value] * empty_len) + if periods > 0: a = empty b = self[:-periods] diff --git a/pandas/tests/arrays/interval/test_interval.py b/pandas/tests/arrays/interval/test_interval.py index 6e72a16fa9b00..7e7762d8973a0 100644 --- a/pandas/tests/arrays/interval/test_interval.py +++ b/pandas/tests/arrays/interval/test_interval.py @@ -89,6 +89,16 @@ def test_shift(self): expected = IntervalArray.from_tuples([(np.nan, np.nan), (1.0, 2.0)]) tm.assert_interval_array_equal(result, expected) + def test_shift_datetime(self): + a = IntervalArray.from_breaks(pd.date_range("2000", periods=4)) + result = a.shift(2) + expected = a.take([-1, -1, 0], allow_fill=True) + tm.assert_interval_array_equal(result, expected) + + result = a.shift(-1) + expected = a.take([1, 2, -1], allow_fill=True) + tm.assert_interval_array_equal(result, expected) + class TestSetitem: def test_set_na(self, left_right_dtypes): From 251165537bc95dec60b35ac845757b24ffafd037 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 3 Feb 2020 11:14:51 -0600 Subject: [PATCH 5/7] fixup --- doc/source/whatsnew/v1.0.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index 4a92a420bd111..11da2dec67998 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -57,7 +57,7 @@ Bug fixes **Interval** -- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers (:issue:`31495`) +- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`) .. --------------------------------------------------------------------------- From 5bea87ca4457c5391fb56351f2322a33578487fd Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 3 Feb 2020 12:03:18 -0600 Subject: [PATCH 6/7] fixup --- doc/source/whatsnew/v1.0.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index 11da2dec67998..64162ae075e3b 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -53,7 +53,7 @@ Bug fixes **Plotting** -- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`** +- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`) **Interval** From 5977d1c3d33cde839ecace2ed1c39a157a95ce09 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 4 Feb 2020 10:18:27 -0600 Subject: [PATCH 7/7] Update doc/source/whatsnew/v1.0.1.rst Co-Authored-By: Joris Van den Bossche --- doc/source/whatsnew/v1.0.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index 64162ae075e3b..be0d144e541b1 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -57,7 +57,7 @@ Bug fixes **Interval** -- Bug in :meth:`arrays.IntervalArray.shift` raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`) +- Bug in :meth:`Series.shift` with ``interval`` dtype raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`) .. ---------------------------------------------------------------------------