diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index d2d5d40393b62..e3e1b35f89cbb 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -281,6 +281,43 @@ that the dates have been converted to UTC .. ipython:: python pd.to_datetime(["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30"], utc=True) +.. _whatsnew_0240.api_breaking.period_end_time: + +Time values in ``dt.end_time`` and ``to_timestamp(how='end')`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The time values in :class:`Period` and :class:`PeriodIndex` objects are now set +to '23:59:59.999999999' when calling :attr:`Series.dt.end_time`, :attr:`Period.end_time`, +:attr:`PeriodIndex.end_time`, :func:`Period.to_timestamp()` with ``how='end'``, +or :func:`PeriodIndex.to_timestamp()` with ``how='end'`` (:issue:`17157`) + +Previous Behavior: + +.. code-block:: ipython + + In [2]: p = pd.Period('2017-01-01', 'D') + In [3]: pi = pd.PeriodIndex([p]) + + In [4]: pd.Series(pi).dt.end_time[0] + Out[4]: Timestamp(2017-01-01 00:00:00) + + In [5]: p.end_time + Out[5]: Timestamp(2017-01-01 23:59:59.999999999) + +Current Behavior: + +Calling :attr:`Series.dt.end_time` will now result in a time of '23:59:59.999999999' as +is the case with :attr:`Period.end_time`, for example + +.. ipython:: python + + p = pd.Period('2017-01-01', 'D') + pi = pd.PeriodIndex([p]) + + pd.Series(pi).dt.end_time[0] + + p.end_time + .. _whatsnew_0240.api.datetimelike.normalize: Tick DateOffset Normalize Restrictions diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index 65fb0f331d039..96d7994bdc822 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -34,6 +34,7 @@ cdef extern from "../src/datetime/np_datetime.h": cimport util from util cimport is_period_object, is_string_object, INT32_MIN +from pandas._libs.tslibs.timedeltas import Timedelta from timestamps import Timestamp from timezones cimport is_utc, is_tzlocal, get_dst_info from timedeltas cimport delta_to_nanoseconds @@ -1221,6 +1222,10 @@ cdef class _Period(object): freq = self._maybe_convert_freq(freq) how = _validate_end_alias(how) + end = how == 'E' + if end: + return (self + 1).to_timestamp(how='start') - Timedelta(1, 'ns') + if freq is None: base, mult = get_freq_code(self.freq) freq = get_to_timestamp_base(base) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 00d53ad82b2dc..26aaab2b1b237 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1235,11 +1235,9 @@ def _generate_regular_range(cls, start, end, periods, freq): tz = None if isinstance(start, Timestamp): tz = start.tz - start = start.to_pydatetime() if isinstance(end, Timestamp): tz = end.tz - end = end.to_pydatetime() xdr = generate_range(start=start, end=end, periods=periods, offset=freq) diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index b315e3ec20830..32aa89010b206 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -25,7 +25,7 @@ from pandas.core.tools.datetimes import parse_time_string from pandas._libs.lib import infer_dtype -from pandas._libs import tslib, index as libindex +from pandas._libs import tslib, index as libindex, Timedelta from pandas._libs.tslibs.period import (Period, IncompatibleFrequency, DIFFERENT_FREQ_INDEX, _validate_end_alias) @@ -501,6 +501,16 @@ def to_timestamp(self, freq=None, how='start'): """ how = _validate_end_alias(how) + end = how == 'E' + if end: + if freq == 'B': + # roll forward to ensure we land on B date + adjust = Timedelta(1, 'D') - Timedelta(1, 'ns') + return self.to_timestamp(how='start') + adjust + else: + adjust = Timedelta(1, 'ns') + return (self + 1).to_timestamp(how='start') - adjust + if freq is None: base, mult = _gfc(self.freq) freq = frequencies.get_to_timestamp_base(base) diff --git a/pandas/tests/frame/test_period.py b/pandas/tests/frame/test_period.py index 482210966fe6b..d56df2371b2e3 100644 --- a/pandas/tests/frame/test_period.py +++ b/pandas/tests/frame/test_period.py @@ -5,7 +5,7 @@ import pandas as pd import pandas.util.testing as tm from pandas import (PeriodIndex, period_range, DataFrame, date_range, - Index, to_datetime, DatetimeIndex) + Index, to_datetime, DatetimeIndex, Timedelta) def _permute(obj): @@ -51,6 +51,7 @@ def test_frame_to_time_stamp(self): df['mix'] = 'a' exp_index = date_range('1/1/2001', end='12/31/2009', freq='A-DEC') + exp_index = exp_index + Timedelta(1, 'D') - Timedelta(1, 'ns') result = df.to_timestamp('D', 'end') tm.assert_index_equal(result.index, exp_index) tm.assert_numpy_array_equal(result.values, df.values) @@ -66,22 +67,26 @@ def _get_with_delta(delta, freq='A-DEC'): delta = timedelta(hours=23) result = df.to_timestamp('H', 'end') exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'h') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) delta = timedelta(hours=23, minutes=59) result = df.to_timestamp('T', 'end') exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'm') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) result = df.to_timestamp('S', 'end') delta = timedelta(hours=23, minutes=59, seconds=59) exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 's') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) # columns df = df.T exp_index = date_range('1/1/2001', end='12/31/2009', freq='A-DEC') + exp_index = exp_index + Timedelta(1, 'D') - Timedelta(1, 'ns') result = df.to_timestamp('D', 'end', axis=1) tm.assert_index_equal(result.columns, exp_index) tm.assert_numpy_array_equal(result.values, df.values) @@ -93,16 +98,19 @@ def _get_with_delta(delta, freq='A-DEC'): delta = timedelta(hours=23) result = df.to_timestamp('H', 'end', axis=1) exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'h') - Timedelta(1, 'ns') tm.assert_index_equal(result.columns, exp_index) delta = timedelta(hours=23, minutes=59) result = df.to_timestamp('T', 'end', axis=1) exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'm') - Timedelta(1, 'ns') tm.assert_index_equal(result.columns, exp_index) result = df.to_timestamp('S', 'end', axis=1) delta = timedelta(hours=23, minutes=59, seconds=59) exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 's') - Timedelta(1, 'ns') tm.assert_index_equal(result.columns, exp_index) # invalid axis diff --git a/pandas/tests/indexes/period/test_period.py b/pandas/tests/indexes/period/test_period.py index 923d826fe1a5e..405edba83dc7a 100644 --- a/pandas/tests/indexes/period/test_period.py +++ b/pandas/tests/indexes/period/test_period.py @@ -366,6 +366,19 @@ def test_periods_number_check(self): with pytest.raises(ValueError): period_range('2011-1-1', '2012-1-1', 'B') + def test_start_time(self): + # GH 17157 + index = PeriodIndex(freq='M', start='2016-01-01', end='2016-05-31') + expected_index = date_range('2016-01-01', end='2016-05-31', freq='MS') + tm.assert_index_equal(index.start_time, expected_index) + + def test_end_time(self): + # GH 17157 + index = PeriodIndex(freq='M', start='2016-01-01', end='2016-05-31') + expected_index = date_range('2016-01-01', end='2016-05-31', freq='M') + expected_index = expected_index.shift(1, freq='D').shift(-1, freq='ns') + tm.assert_index_equal(index.end_time, expected_index) + def test_index_duplicate_periods(self): # monotonic idx = PeriodIndex([2000, 2007, 2007, 2009, 2009], freq='A-JUN') diff --git a/pandas/tests/indexes/period/test_scalar_compat.py b/pandas/tests/indexes/period/test_scalar_compat.py index 56bd2adf58719..a66a81fe99cd4 100644 --- a/pandas/tests/indexes/period/test_scalar_compat.py +++ b/pandas/tests/indexes/period/test_scalar_compat.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Tests for PeriodIndex behaving like a vectorized Period scalar""" -from pandas import PeriodIndex, date_range +from pandas import PeriodIndex, date_range, Timedelta import pandas.util.testing as tm @@ -14,4 +14,5 @@ def test_start_time(self): def test_end_time(self): index = PeriodIndex(freq='M', start='2016-01-01', end='2016-05-31') expected_index = date_range('2016-01-01', end='2016-05-31', freq='M') + expected_index += Timedelta(1, 'D') - Timedelta(1, 'ns') tm.assert_index_equal(index.end_time, expected_index) diff --git a/pandas/tests/indexes/period/test_tools.py b/pandas/tests/indexes/period/test_tools.py index 16b558916df2d..c4ed07d98413f 100644 --- a/pandas/tests/indexes/period/test_tools.py +++ b/pandas/tests/indexes/period/test_tools.py @@ -3,6 +3,7 @@ import pytest import pandas as pd +from pandas import Timedelta import pandas.util.testing as tm import pandas.core.indexes.period as period from pandas.compat import lrange @@ -60,6 +61,7 @@ def test_to_timestamp(self): exp_index = date_range('1/1/2001', end='12/31/2009', freq='A-DEC') result = series.to_timestamp(how='end') + exp_index = exp_index + Timedelta(1, 'D') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) assert result.name == 'foo' @@ -74,16 +76,19 @@ def _get_with_delta(delta, freq='A-DEC'): delta = timedelta(hours=23) result = series.to_timestamp('H', 'end') exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'h') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) delta = timedelta(hours=23, minutes=59) result = series.to_timestamp('T', 'end') exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 'm') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) result = series.to_timestamp('S', 'end') delta = timedelta(hours=23, minutes=59, seconds=59) exp_index = _get_with_delta(delta) + exp_index = exp_index + Timedelta(1, 's') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) index = PeriodIndex(freq='H', start='1/1/2001', end='1/2/2001') @@ -92,6 +97,7 @@ def _get_with_delta(delta, freq='A-DEC'): exp_index = date_range('1/1/2001 00:59:59', end='1/2/2001 00:59:59', freq='H') result = series.to_timestamp(how='end') + exp_index = exp_index + Timedelta(1, 's') - Timedelta(1, 'ns') tm.assert_index_equal(result.index, exp_index) assert result.name == 'foo' @@ -284,6 +290,7 @@ def test_to_timestamp_pi_mult(self): result = idx.to_timestamp(how='E') expected = DatetimeIndex(['2011-02-28', 'NaT', '2011-03-31'], name='idx') + expected = expected + Timedelta(1, 'D') - Timedelta(1, 'ns') tm.assert_index_equal(result, expected) def test_to_timestamp_pi_combined(self): @@ -298,11 +305,13 @@ def test_to_timestamp_pi_combined(self): expected = DatetimeIndex(['2011-01-02 00:59:59', '2011-01-03 01:59:59'], name='idx') + expected = expected + Timedelta(1, 's') - Timedelta(1, 'ns') tm.assert_index_equal(result, expected) result = idx.to_timestamp(how='E', freq='H') expected = DatetimeIndex(['2011-01-02 00:00', '2011-01-03 01:00'], name='idx') + expected = expected + Timedelta(1, 'h') - Timedelta(1, 'ns') tm.assert_index_equal(result, expected) def test_period_astype_to_timestamp(self): @@ -312,6 +321,7 @@ def test_period_astype_to_timestamp(self): tm.assert_index_equal(pi.astype('datetime64[ns]'), exp) exp = pd.DatetimeIndex(['2011-01-31', '2011-02-28', '2011-03-31']) + exp = exp + Timedelta(1, 'D') - Timedelta(1, 'ns') tm.assert_index_equal(pi.astype('datetime64[ns]', how='end'), exp) exp = pd.DatetimeIndex(['2011-01-01', '2011-02-01', '2011-03-01'], @@ -321,6 +331,7 @@ def test_period_astype_to_timestamp(self): exp = pd.DatetimeIndex(['2011-01-31', '2011-02-28', '2011-03-31'], tz='US/Eastern') + exp = exp + Timedelta(1, 'D') - Timedelta(1, 'ns') res = pi.astype('datetime64[ns, US/Eastern]', how='end') tm.assert_index_equal(res, exp) diff --git a/pandas/tests/scalar/period/test_period.py b/pandas/tests/scalar/period/test_period.py index eccd86a888fb9..4a17b2efd1dec 100644 --- a/pandas/tests/scalar/period/test_period.py +++ b/pandas/tests/scalar/period/test_period.py @@ -5,6 +5,7 @@ from datetime import datetime, date, timedelta import pandas as pd +from pandas import Timedelta import pandas.util.testing as tm import pandas.core.indexes.period as period from pandas.compat import text_type, iteritems @@ -274,12 +275,14 @@ def test_timestamp_tz_arg_dateutil_from_string(self): def test_timestamp_mult(self): p = pd.Period('2011-01', freq='M') - assert p.to_timestamp(how='S') == pd.Timestamp('2011-01-01') - assert p.to_timestamp(how='E') == pd.Timestamp('2011-01-31') + assert p.to_timestamp(how='S') == Timestamp('2011-01-01') + expected = Timestamp('2011-02-01') - Timedelta(1, 'ns') + assert p.to_timestamp(how='E') == expected p = pd.Period('2011-01', freq='3M') - assert p.to_timestamp(how='S') == pd.Timestamp('2011-01-01') - assert p.to_timestamp(how='E') == pd.Timestamp('2011-03-31') + assert p.to_timestamp(how='S') == Timestamp('2011-01-01') + expected = Timestamp('2011-04-01') - Timedelta(1, 'ns') + assert p.to_timestamp(how='E') == expected def test_construction(self): i1 = Period('1/1/2005', freq='M') @@ -611,19 +614,19 @@ def _ex(p): p = Period('1985', freq='A') result = p.to_timestamp('H', how='end') - expected = datetime(1985, 12, 31, 23) + expected = Timestamp(1986, 1, 1) - Timedelta(1, 'ns') assert result == expected result = p.to_timestamp('3H', how='end') assert result == expected result = p.to_timestamp('T', how='end') - expected = datetime(1985, 12, 31, 23, 59) + expected = Timestamp(1986, 1, 1) - Timedelta(1, 'ns') assert result == expected result = p.to_timestamp('2T', how='end') assert result == expected result = p.to_timestamp(how='end') - expected = datetime(1985, 12, 31) + expected = Timestamp(1986, 1, 1) - Timedelta(1, 'ns') assert result == expected expected = datetime(1985, 1, 1) diff --git a/pandas/tests/series/test_period.py b/pandas/tests/series/test_period.py index 63726f27914f3..90dbe26a2f0ea 100644 --- a/pandas/tests/series/test_period.py +++ b/pandas/tests/series/test_period.py @@ -3,7 +3,8 @@ import pandas as pd import pandas.util.testing as tm import pandas.core.indexes.period as period -from pandas import Series, period_range, DataFrame +from pandas import Series, period_range, DataFrame, Period +import pytest def _permute(obj): @@ -167,3 +168,23 @@ def test_truncate(self): pd.Period('2017-09-02') ]) tm.assert_series_equal(result2, pd.Series([2], index=expected_idx2)) + + @pytest.mark.parametrize('input_vals', [ + [Period('2016-01', freq='M'), Period('2016-02', freq='M')], + [Period('2016-01-01', freq='D'), Period('2016-01-02', freq='D')], + [Period('2016-01-01 00:00:00', freq='H'), + Period('2016-01-01 01:00:00', freq='H')], + [Period('2016-01-01 00:00:00', freq='M'), + Period('2016-01-01 00:01:00', freq='M')], + [Period('2016-01-01 00:00:00', freq='S'), + Period('2016-01-01 00:00:01', freq='S')] + ]) + def test_end_time_timevalues(self, input_vals): + # GH 17157 + # Check that the time part of the Period is adjusted by end_time + # when using the dt accessor on a Series + + s = Series(input_vals) + result = s.dt.end_time + expected = s.apply(lambda x: x.end_time) + tm.assert_series_equal(result, expected) diff --git a/pandas/tests/test_resample.py b/pandas/tests/test_resample.py index 1f70d09e43b37..de4dc2bcf25a4 100644 --- a/pandas/tests/test_resample.py +++ b/pandas/tests/test_resample.py @@ -21,7 +21,7 @@ import pandas as pd from pandas import (Series, DataFrame, Panel, Index, isna, - notna, Timestamp) + notna, Timestamp, Timedelta) from pandas.compat import range, lrange, zip, OrderedDict from pandas.errors import UnsupportedFunctionCall @@ -1702,12 +1702,14 @@ def test_resample_anchored_intraday(self): result = df.resample('M').mean() expected = df.resample( 'M', kind='period').mean().to_timestamp(how='end') + expected.index += Timedelta(1, 'ns') - Timedelta(1, 'D') tm.assert_frame_equal(result, expected) result = df.resample('M', closed='left').mean() exp = df.tshift(1, freq='D').resample('M', kind='period').mean() exp = exp.to_timestamp(how='end') + exp.index = exp.index + Timedelta(1, 'ns') - Timedelta(1, 'D') tm.assert_frame_equal(result, exp) rng = date_range('1/1/2012', '4/1/2012', freq='100min') @@ -1716,12 +1718,14 @@ def test_resample_anchored_intraday(self): result = df.resample('Q').mean() expected = df.resample( 'Q', kind='period').mean().to_timestamp(how='end') + expected.index += Timedelta(1, 'ns') - Timedelta(1, 'D') tm.assert_frame_equal(result, expected) result = df.resample('Q', closed='left').mean() expected = df.tshift(1, freq='D').resample('Q', kind='period', closed='left').mean() expected = expected.to_timestamp(how='end') + expected.index += Timedelta(1, 'ns') - Timedelta(1, 'D') tm.assert_frame_equal(result, expected) ts = _simple_ts('2012-04-29 23:00', '2012-04-30 5:00', freq='h') @@ -2473,7 +2477,7 @@ def test_resample_to_timestamps(self): ts = _simple_pts('1/1/1990', '12/31/1995', freq='M') result = ts.resample('A-DEC', kind='timestamp').mean() - expected = ts.to_timestamp(how='end').resample('A-DEC').mean() + expected = ts.to_timestamp(how='start').resample('A-DEC').mean() assert_series_equal(result, expected) def test_resample_to_quarterly(self): diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 60981f41ec716..9d41401a7eefc 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -1321,7 +1321,7 @@ def _end_apply_index(self, dtindex): roll = self.n base = (base_period + roll).to_timestamp(how='end') - return base + off + return base + off + Timedelta(1, 'ns') - Timedelta(1, 'D') def onOffset(self, dt): if self.normalize and not _is_normalized(dt):