From 85a72d7cb6097ab6fca07d28e4a0f390ee78ea08 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 11 Jul 2019 23:28:19 -0700 Subject: [PATCH 1/5] BUG: Series.__setitem__ with tuple and tz aware data --- doc/source/whatsnew/v0.25.0.rst | 3 ++- pandas/core/series.py | 4 ++++ pandas/tests/series/indexing/test_indexing.py | 11 +++++++++++ pandas/tests/series/test_timezones.py | 10 +++++++++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index eeaafd7ad7d51..e68595e132222 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -995,6 +995,7 @@ Timezones - Bug in :func:`DataFrame.join` where joining a timezone aware index with a timezone aware column would result in a column of ``NaN`` (:issue:`26335`) - Bug in :func:`date_range` where ambiguous or nonexistent start or end times were not handled by the ``ambiguous`` or ``nonexistent`` keywords respectively (:issue:`27088`) - Bug in :meth:`DatetimeIndex.union` when combining a timezone aware and timezone unaware :class:`DatetimeIndex` (:issue:`21671`) +- Bug when applying a numpy reduction function (e.g. ``np.minimum``) to a timezone aware :class:`Series` (:issue:`15552`) Numeric ^^^^^^^ @@ -1054,7 +1055,7 @@ Indexing - Bug in :class:`CategoricalIndex` and :class:`Categorical` incorrectly raising ``ValueError`` instead of ``TypeError`` when a list is passed using the ``in`` operator (``__contains__``) (:issue:`21729`) - Bug in setting a new value in a :class:`Series` with a :class:`Timedelta` object incorrectly casting the value to an integer (:issue:`22717`) - Bug in :class:`Series` setting a new key (``__setitem__``) with a timezone-aware datetime incorrectly raising ``ValueError`` (:issue:`12862`) -- +- Bug in :class:`Series` setting a existing tuple key (``__setitem__``) with timezone-aware datetime values incorrectly raising ``TypeError`` (:issue:`20441`) Missing ^^^^^^^ diff --git a/pandas/core/series.py b/pandas/core/series.py index acb0826953508..392b1bd5471d1 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1250,6 +1250,10 @@ def setitem(key, value): def _set_with_engine(self, key, value): values = self._values try: + if is_extension_array_dtype(values): + # The cython indexing routines do not support ExtensionArrays. + # Defer to the next setting routine. + raise KeyError self.index._engine.set_value(values, key, value) return except KeyError: diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 6ff878f07da84..b329cd7e38148 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -847,3 +847,14 @@ def test_head_tail(test_data): assert_series_equal(test_data.series.head(0), test_data.series[0:0]) assert_series_equal(test_data.series.tail(), test_data.series[-5:]) assert_series_equal(test_data.series.tail(0), test_data.series[0:0]) + + +def test_setitem_tuple_with_datetimetz(): + # GH 20441 + arr = pd.date_range('2017', periods=4, tz='US/Eastern') + index = [(0, 1), (0, 2), (0, 3), (0, 4)] + result = Series(arr, index=index) + expected = result.copy() + result[(0, 1)] = np.nan + expected.iloc[0] = np.nan + assert_series_equal(result, expected) diff --git a/pandas/tests/series/test_timezones.py b/pandas/tests/series/test_timezones.py index c16e2864b131f..f8754e0c3bf43 100644 --- a/pandas/tests/series/test_timezones.py +++ b/pandas/tests/series/test_timezones.py @@ -10,7 +10,7 @@ from pandas._libs.tslibs import conversion, timezones -from pandas import DatetimeIndex, Index, NaT, Series, Timestamp +from pandas import DatetimeIndex, Index, NaT, Series, Timestamp, to_datetime from pandas.core.indexes.datetimes import date_range import pandas.util.testing as tm @@ -379,3 +379,11 @@ def test_constructor_data_aware_dtype_naive(self, tz_aware_fixture): result = Series([Timestamp("2019", tz=tz)], dtype="datetime64[ns]") expected = Series([Timestamp("2019")]) tm.assert_series_equal(result, expected) + + def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture): + # GH 15552 + tz = tz_aware_fixture + arg = to_datetime(['2019']).tz_localize(tz) + expected = Series(arg) + result = np.minimum(expected, expected) + tm.assert_series_equal(result, expected) From 4be5326096ab8408fc7d095eb6829037e2853e16 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 12 Jul 2019 13:52:07 -0700 Subject: [PATCH 2/5] Fix plotting DatetimeIndex with datetime.timezone.utc objects --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/plotting/_matplotlib/converter.py | 3 --- pandas/tests/plotting/test_datetimelike.py | 7 ++++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index e68595e132222..cfdc155d7062f 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -1112,7 +1112,7 @@ Plotting - Bug in an error message in :meth:`DataFrame.plot`. Improved the error message if non-numerics are passed to :meth:`DataFrame.plot` (:issue:`25481`) - Bug in incorrect ticklabel positions when plotting an index that are non-numeric / non-datetime (:issue:`7612`, :issue:`15912`, :issue:`22334`) - Fixed bug causing plots of :class:`PeriodIndex` timeseries to fail if the frequency is a multiple of the frequency rule code (:issue:`14763`) -- +- Fixed bug when plotting a :class:`DatetimeIndex` with ``datetime.timezone.utc`` timezone (:issue:`17173`) - - diff --git a/pandas/plotting/_matplotlib/converter.py b/pandas/plotting/_matplotlib/converter.py index b20dd3212c7cb..15648d59c8f98 100644 --- a/pandas/plotting/_matplotlib/converter.py +++ b/pandas/plotting/_matplotlib/converter.py @@ -324,9 +324,6 @@ def axisinfo(unit, axis): class PandasAutoDateFormatter(dates.AutoDateFormatter): def __init__(self, locator, tz=None, defaultfmt="%Y-%m-%d"): dates.AutoDateFormatter.__init__(self, locator, tz, defaultfmt) - # matplotlib.dates._UTC has no _utcoffset called by pandas - if self._tz is dates.UTC: - self._tz._utcoffset = self._tz.utcoffset(None) class PandasAutoDateLocator(dates.AutoDateLocator): diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index 5ae29dc640dc9..e3bc3d452f038 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -45,9 +45,10 @@ def teardown_method(self, method): tm.close() @pytest.mark.slow - def test_ts_plot_with_tz(self): - # GH2877 - index = date_range("1/1/2011", periods=2, freq="H", tz="Europe/Brussels") + def test_ts_plot_with_tz(self, tz_aware_fixture): + # GH2877, GH17173 + tz = tz_aware_fixture + index = date_range("1/1/2011", periods=2, freq="H", tz=tz) ts = Series([188.5, 328.25], index=index) _check_plot_works(ts.plot) From 14b09c6ec68bd2f769137c718e5db81f39d980d8 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 12 Jul 2019 13:53:06 -0700 Subject: [PATCH 3/5] black --- pandas/tests/series/indexing/test_indexing.py | 2 +- pandas/tests/series/test_timezones.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index b329cd7e38148..c85377b5a735c 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -851,7 +851,7 @@ def test_head_tail(test_data): def test_setitem_tuple_with_datetimetz(): # GH 20441 - arr = pd.date_range('2017', periods=4, tz='US/Eastern') + arr = pd.date_range("2017", periods=4, tz="US/Eastern") index = [(0, 1), (0, 2), (0, 3), (0, 4)] result = Series(arr, index=index) expected = result.copy() diff --git a/pandas/tests/series/test_timezones.py b/pandas/tests/series/test_timezones.py index f8754e0c3bf43..523c988b8998a 100644 --- a/pandas/tests/series/test_timezones.py +++ b/pandas/tests/series/test_timezones.py @@ -383,7 +383,7 @@ def test_constructor_data_aware_dtype_naive(self, tz_aware_fixture): def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture): # GH 15552 tz = tz_aware_fixture - arg = to_datetime(['2019']).tz_localize(tz) + arg = to_datetime(["2019"]).tz_localize(tz) expected = Series(arg) result = np.minimum(expected, expected) tm.assert_series_equal(result, expected) From a2af6100e97dc1d6f8111c3ab397a2086e160036 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 12 Jul 2019 18:07:10 -0700 Subject: [PATCH 4/5] Address comments --- pandas/core/series.py | 6 +++--- pandas/tests/reductions/test_reductions.py | 8 ++++++++ pandas/tests/series/test_timezones.py | 10 +--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index 392b1bd5471d1..a70e6b4b3cab4 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1251,9 +1251,9 @@ def _set_with_engine(self, key, value): values = self._values try: if is_extension_array_dtype(values): - # The cython indexing routines do not support ExtensionArrays. - # Defer to the next setting routine. - raise KeyError + # The cython indexing engine does not support ExtensionArrays. + values[self.index.get_loc(key)] = value + return self.index._engine.set_value(values, key, value) return except KeyError: diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index 1e7a40b9040b7..025d690a3a01a 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -159,6 +159,14 @@ def test_same_tz_min_max_axis_1(self, op, expected_col): expected = df[expected_col].rename(None) tm.assert_series_equal(result, expected) + def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture): + # GH 15552 + tz = tz_aware_fixture + arg = pd.to_datetime(["2019"]).tz_localize(tz) + expected = Series(arg) + result = np.minimum(expected, expected) + tm.assert_series_equal(result, expected) + class TestIndexReductions: # Note: the name TestIndexReductions indicates these tests diff --git a/pandas/tests/series/test_timezones.py b/pandas/tests/series/test_timezones.py index 523c988b8998a..c16e2864b131f 100644 --- a/pandas/tests/series/test_timezones.py +++ b/pandas/tests/series/test_timezones.py @@ -10,7 +10,7 @@ from pandas._libs.tslibs import conversion, timezones -from pandas import DatetimeIndex, Index, NaT, Series, Timestamp, to_datetime +from pandas import DatetimeIndex, Index, NaT, Series, Timestamp from pandas.core.indexes.datetimes import date_range import pandas.util.testing as tm @@ -379,11 +379,3 @@ def test_constructor_data_aware_dtype_naive(self, tz_aware_fixture): result = Series([Timestamp("2019", tz=tz)], dtype="datetime64[ns]") expected = Series([Timestamp("2019")]) tm.assert_series_equal(result, expected) - - def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture): - # GH 15552 - tz = tz_aware_fixture - arg = to_datetime(["2019"]).tz_localize(tz) - expected = Series(arg) - result = np.minimum(expected, expected) - tm.assert_series_equal(result, expected) From e7bc38170f162d78b7019318de24f478c25e7d72 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 15 Jul 2019 15:50:37 -0700 Subject: [PATCH 5/5] Address comments --- doc/source/whatsnew/v0.25.0.rst | 4 ++-- pandas/core/series.py | 8 ++++---- pandas/tests/reductions/test_reductions.py | 5 +++-- pandas/tests/series/indexing/test_datetime.py | 11 +++++++++++ pandas/tests/series/indexing/test_indexing.py | 11 ----------- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index cd1fe55327452..38c579e9518a0 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -995,7 +995,7 @@ Timezones - Bug in :func:`DataFrame.join` where joining a timezone aware index with a timezone aware column would result in a column of ``NaN`` (:issue:`26335`) - Bug in :func:`date_range` where ambiguous or nonexistent start or end times were not handled by the ``ambiguous`` or ``nonexistent`` keywords respectively (:issue:`27088`) - Bug in :meth:`DatetimeIndex.union` when combining a timezone aware and timezone unaware :class:`DatetimeIndex` (:issue:`21671`) -- Bug when applying a numpy reduction function (e.g. ``np.minimum``) to a timezone aware :class:`Series` (:issue:`15552`) +- Bug when applying a numpy reduction function (e.g. :meth:`numpy.minimum`) to a timezone aware :class:`Series` (:issue:`15552`) Numeric ^^^^^^^ @@ -1056,7 +1056,7 @@ Indexing - Bug in setting a new value in a :class:`Series` with a :class:`Timedelta` object incorrectly casting the value to an integer (:issue:`22717`) - Bug in :class:`Series` setting a new key (``__setitem__``) with a timezone-aware datetime incorrectly raising ``ValueError`` (:issue:`12862`) - Bug in :meth:`DataFrame.iloc` when indexing with a read-only indexer (:issue:`17192`) -- Bug in :class:`Series` setting a existing tuple key (``__setitem__``) with timezone-aware datetime values incorrectly raising ``TypeError`` (:issue:`20441`) +- Bug in :class:`Series` setting an existing tuple key (``__setitem__``) with timezone-aware datetime values incorrectly raising ``TypeError`` (:issue:`20441`) Missing ^^^^^^^ diff --git a/pandas/core/series.py b/pandas/core/series.py index cd072fb1f9039..46b96c1ece77c 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1236,11 +1236,11 @@ def setitem(key, value): def _set_with_engine(self, key, value): values = self._values + if is_extension_array_dtype(values.dtype): + # The cython indexing engine does not support ExtensionArrays. + values[self.index.get_loc(key)] = value + return try: - if is_extension_array_dtype(values): - # The cython indexing engine does not support ExtensionArrays. - values[self.index.get_loc(key)] = value - return self.index._engine.set_value(values, key, value) return except KeyError: diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index 025d690a3a01a..05ebff4387908 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -159,12 +159,13 @@ def test_same_tz_min_max_axis_1(self, op, expected_col): expected = df[expected_col].rename(None) tm.assert_series_equal(result, expected) - def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture): + @pytest.mark.parametrize("func", ["maximum", "minimum"]) + def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture, func): # GH 15552 tz = tz_aware_fixture arg = pd.to_datetime(["2019"]).tz_localize(tz) expected = Series(arg) - result = np.minimum(expected, expected) + result = getattr(np, func)(expected, expected) tm.assert_series_equal(result, expected) diff --git a/pandas/tests/series/indexing/test_datetime.py b/pandas/tests/series/indexing/test_datetime.py index e2f40c6267493..e0b84e8708fa1 100644 --- a/pandas/tests/series/indexing/test_datetime.py +++ b/pandas/tests/series/indexing/test_datetime.py @@ -765,3 +765,14 @@ def test_round_nat(method, freq): expected = Series(pd.NaT) round_method = getattr(s.dt, method) assert_series_equal(round_method(freq), expected) + + +def test_setitem_tuple_with_datetimetz(): + # GH 20441 + arr = date_range("2017", periods=4, tz="US/Eastern") + index = [(0, 1), (0, 2), (0, 3), (0, 4)] + result = Series(arr, index=index) + expected = result.copy() + result[(0, 1)] = np.nan + expected.iloc[0] = np.nan + assert_series_equal(result, expected) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c85377b5a735c..6ff878f07da84 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -847,14 +847,3 @@ def test_head_tail(test_data): assert_series_equal(test_data.series.head(0), test_data.series[0:0]) assert_series_equal(test_data.series.tail(), test_data.series[-5:]) assert_series_equal(test_data.series.tail(0), test_data.series[0:0]) - - -def test_setitem_tuple_with_datetimetz(): - # GH 20441 - arr = pd.date_range("2017", periods=4, tz="US/Eastern") - index = [(0, 1), (0, 2), (0, 3), (0, 4)] - result = Series(arr, index=index) - expected = result.copy() - result[(0, 1)] = np.nan - expected.iloc[0] = np.nan - assert_series_equal(result, expected)