From d83af3f5b996383b5ff6d0bbcea3abf46ebe157c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 15 Jan 2020 20:31:58 -0800 Subject: [PATCH 1/2] BUG: partial-timestamp slicing near the end of year/quarter/month --- doc/source/whatsnew/v1.1.0.rst | 2 +- pandas/core/indexes/datetimes.py | 27 ++++++++++++------- .../indexes/datetimes/test_partial_slicing.py | 19 +++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index b5a7b19f160a4..c848672e47dfa 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -102,7 +102,7 @@ Interval Indexing ^^^^^^^^ - +- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`????`) - - diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 23ced8987d8ac..1c24a566b13a2 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -5,7 +5,14 @@ import numpy as np -from pandas._libs import NaT, Timestamp, index as libindex, lib, tslib as libts +from pandas._libs import ( + NaT, + Timedelta, + Timestamp, + index as libindex, + lib, + tslib as libts, +) from pandas._libs.tslibs import ccalendar, fields, parsing, timezones from pandas.util._decorators import Appender, Substitution, cache_readonly @@ -31,7 +38,7 @@ import pandas.core.tools.datetimes as tools from pandas.tseries.frequencies import Resolution, to_offset -from pandas.tseries.offsets import Nano, prefix_mapping +from pandas.tseries.offsets import prefix_mapping def _new_DatetimeIndex(cls, d): @@ -519,27 +526,27 @@ def _parsed_string_to_bounds(self, reso, parsed): raise KeyError if reso == "year": start = Timestamp(parsed.year, 1, 1) - end = Timestamp(parsed.year, 12, 31, 23, 59, 59, 999999) + end = Timestamp(parsed.year + 1, 1, 1) - Timedelta(nanoseconds=1) elif reso == "month": d = ccalendar.get_days_in_month(parsed.year, parsed.month) start = Timestamp(parsed.year, parsed.month, 1) - end = Timestamp(parsed.year, parsed.month, d, 23, 59, 59, 999999) + end = start + Timedelta(days=d, nanoseconds=-1) elif reso == "quarter": qe = (((parsed.month - 1) + 2) % 12) + 1 # two months ahead d = ccalendar.get_days_in_month(parsed.year, qe) # at end of month start = Timestamp(parsed.year, parsed.month, 1) - end = Timestamp(parsed.year, qe, d, 23, 59, 59, 999999) + end = Timestamp(parsed.year, qe, 1) + Timedelta(days=d, nanoseconds=-1) elif reso == "day": start = Timestamp(parsed.year, parsed.month, parsed.day) - end = start + timedelta(days=1) - Nano(1) + end = start + Timedelta(days=1, nanoseconds=-1) elif reso == "hour": start = Timestamp(parsed.year, parsed.month, parsed.day, parsed.hour) - end = start + timedelta(hours=1) - Nano(1) + end = start + Timedelta(hours=1, nanoseconds=-1) elif reso == "minute": start = Timestamp( parsed.year, parsed.month, parsed.day, parsed.hour, parsed.minute ) - end = start + timedelta(minutes=1) - Nano(1) + end = start + Timedelta(minutes=1, nanoseconds=-1) elif reso == "second": start = Timestamp( parsed.year, @@ -549,7 +556,7 @@ def _parsed_string_to_bounds(self, reso, parsed): parsed.minute, parsed.second, ) - end = start + timedelta(seconds=1) - Nano(1) + end = start + Timedelta(seconds=1, nanoseconds=-1) elif reso == "microsecond": start = Timestamp( parsed.year, @@ -560,7 +567,7 @@ def _parsed_string_to_bounds(self, reso, parsed): parsed.second, parsed.microsecond, ) - end = start + timedelta(microseconds=1) - Nano(1) + end = start + Timedelta(microseconds=1, nanoseconds=-1) # GH 24076 # If an incoming date string contained a UTC offset, need to localize # the parsed date to this offset first before aligning with the index's diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py index e30cc4449e01e..1cb87ecf25026 100644 --- a/pandas/tests/indexes/datetimes/test_partial_slicing.py +++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py @@ -142,6 +142,25 @@ def test_slice_year(self): expected = slice(3288, 3653) assert result == expected + @pytest.mark.parametrize( + "partial_dtime", + [ + "2019", + "2019Q4", + "Dec 2019", + "2019-12-31", + "2019-12-31 23", + "2019-12-31 23:59", + ], + ) + def test_slice_end_of_period_resolution(self, partial_dtime): + dti = date_range("2019-12-31 23:59:55.999999999", periods=10, freq="s") + + ser = pd.Series(range(10), index=dti) + result = ser[partial_dtime] + expected = ser.iloc[:5] + tm.assert_series_equal(result, expected) + def test_slice_quarter(self): dti = date_range(freq="D", start=datetime(2000, 6, 1), periods=500) From 1f51ed8533e354e517fc4ada6313d51056dcd988 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 15 Jan 2020 20:33:31 -0800 Subject: [PATCH 2/2] add GH ref --- doc/source/whatsnew/v1.1.0.rst | 2 +- pandas/tests/indexes/datetimes/test_partial_slicing.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index c848672e47dfa..4e8cc3ce1d04f 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -102,7 +102,7 @@ Interval Indexing ^^^^^^^^ -- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`????`) +- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`31064`) - - diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py index 1cb87ecf25026..946d658e90132 100644 --- a/pandas/tests/indexes/datetimes/test_partial_slicing.py +++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py @@ -154,6 +154,7 @@ def test_slice_year(self): ], ) def test_slice_end_of_period_resolution(self, partial_dtime): + # GH#31064 dti = date_range("2019-12-31 23:59:55.999999999", periods=10, freq="s") ser = pd.Series(range(10), index=dti)