From cff6ee6919ec90e573793a72d50b3c55aa98d1c1 Mon Sep 17 00:00:00 2001 From: deponovo Date: Wed, 29 Dec 2021 10:47:58 +0100 Subject: [PATCH 1/7] BUG: now any nanoseconds contribution will be properly considered in the timedelta constructor BUG: overriden the Timedelta.total_seconds() to return the correct value containing also the nanoseconds portion --- pandas/_libs/tslibs/timedeltas.pyx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 2ac635ad4fb9c..88f39819d2b03 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -1268,7 +1268,16 @@ class Timedelta(_Timedelta): kwargs = {key: _to_py_int_float(kwargs[key]) for key in kwargs} - nano = convert_to_timedelta64(kwargs.pop('nanoseconds', 0), 'ns') + # GH43764, making sure any nanoseconds contributions from any kwarg is taken into consideration + nano = convert_to_timedelta64( + ( + kwargs.pop('nanoseconds', 0) + + kwargs.pop('microseconds', 0) * 1000 + + kwargs.pop('milliseconds', 0) * 1000000 + + kwargs.pop('seconds', 0) * 1000000000 + ), 'ns' + ) + try: value = nano + convert_to_timedelta64(timedelta(**kwargs), 'ns') @@ -1520,6 +1529,10 @@ class Timedelta(_Timedelta): div = other // self return div, other - div * self + # GH40946 + def total_seconds(self): + return self.value / 1e9 + cdef bint is_any_td_scalar(object obj): """ From c0f8127c03b8a8eddcf312940cd0c728b7aa9543 Mon Sep 17 00:00:00 2001 From: deponovo Date: Wed, 29 Dec 2021 10:58:36 +0100 Subject: [PATCH 2/7] TST: extended tests to cover new timdelta construction possibilities --- pandas/tests/tslibs/test_timedeltas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/tslibs/test_timedeltas.py b/pandas/tests/tslibs/test_timedeltas.py index 25450bd64a298..f94596d88ab78 100644 --- a/pandas/tests/tslibs/test_timedeltas.py +++ b/pandas/tests/tslibs/test_timedeltas.py @@ -15,6 +15,9 @@ (np.timedelta64(14, "D"), 14 * 24 * 3600 * 1e9), (Timedelta(minutes=-7), -7 * 60 * 1e9), (Timedelta(minutes=-7).to_pytimedelta(), -7 * 60 * 1e9), + (Timedelta(seconds=1234e-9), 1234), # GH43764, GH40946 + (Timedelta(seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 111), # GH43764, GH40946 + (Timedelta(days=1, seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 24 * 3600e9 + 111), # GH43764, GH40946 (offsets.Nano(125), 125), (1, 1), (np.int64(2), 2), From c9d77477c12ad8b056dab5ca9891ebb91457de77 Mon Sep 17 00:00:00 2001 From: deponovo Date: Wed, 29 Dec 2021 11:07:07 +0100 Subject: [PATCH 3/7] CLN: pre-commit cleanup --- pandas/tests/tslibs/test_timedeltas.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tslibs/test_timedeltas.py b/pandas/tests/tslibs/test_timedeltas.py index f94596d88ab78..0162d09cb5d7c 100644 --- a/pandas/tests/tslibs/test_timedeltas.py +++ b/pandas/tests/tslibs/test_timedeltas.py @@ -16,8 +16,14 @@ (Timedelta(minutes=-7), -7 * 60 * 1e9), (Timedelta(minutes=-7).to_pytimedelta(), -7 * 60 * 1e9), (Timedelta(seconds=1234e-9), 1234), # GH43764, GH40946 - (Timedelta(seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 111), # GH43764, GH40946 - (Timedelta(days=1, seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 24 * 3600e9 + 111), # GH43764, GH40946 + ( + Timedelta(seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), + 111, + ), # GH43764, GH40946 + ( + Timedelta(days=1, seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), + 24 * 3600e9 + 111, + ), # GH43764, GH40946 (offsets.Nano(125), 125), (1, 1), (np.int64(2), 2), From e1110bf977157a4a15d862ffead972e8d5231175 Mon Sep 17 00:00:00 2001 From: deponovo Date: Wed, 29 Dec 2021 11:12:10 +0100 Subject: [PATCH 4/7] DOC: updated the whatsnew --- doc/source/whatsnew/v1.4.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 2c77b87f8efb6..92232d09a7a26 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -236,6 +236,8 @@ Other enhancements - :meth:`is_list_like` now identifies duck-arrays as list-like unless ``.ndim == 0`` (:issue:`35131`) - :class:`ExtensionDtype` and :class:`ExtensionArray` are now (de)serialized when exporting a :class:`DataFrame` with :meth:`DataFrame.to_json` using ``orient='table'`` (:issue:`20612`, :issue:`44705`). - Add support for `Zstandard `_ compression to :meth:`DataFrame.to_pickle`/:meth:`read_pickle` and friends (:issue:`43925`) +- :class:`Timedelta` now properly taking into account any nanoseconds contribution (:issue: `43764`) +- :meth:`Timedelta.total_seconds()` now properly taking into account any nanoseconds contribution (:issue: `40946`) - From 4406deb21cc3a58e56ebaaf13f983833dfcdc0c3 Mon Sep 17 00:00:00 2001 From: deponovo Date: Thu, 30 Dec 2021 16:06:34 +0100 Subject: [PATCH 5/7] CLN: removed GH40946 related modification --- doc/source/whatsnew/v1.4.0.rst | 3 +-- pandas/_libs/tslibs/timedeltas.pyx | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 92232d09a7a26..27b205334ed4d 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -236,8 +236,7 @@ Other enhancements - :meth:`is_list_like` now identifies duck-arrays as list-like unless ``.ndim == 0`` (:issue:`35131`) - :class:`ExtensionDtype` and :class:`ExtensionArray` are now (de)serialized when exporting a :class:`DataFrame` with :meth:`DataFrame.to_json` using ``orient='table'`` (:issue:`20612`, :issue:`44705`). - Add support for `Zstandard `_ compression to :meth:`DataFrame.to_pickle`/:meth:`read_pickle` and friends (:issue:`43925`) -- :class:`Timedelta` now properly taking into account any nanoseconds contribution (:issue: `43764`) -- :meth:`Timedelta.total_seconds()` now properly taking into account any nanoseconds contribution (:issue: `40946`) +- :class:`Timedelta` now properly taking into account any nanoseconds contribution (:issue:`43764`) - diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 88f39819d2b03..8fab67eb7db16 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -1268,7 +1268,8 @@ class Timedelta(_Timedelta): kwargs = {key: _to_py_int_float(kwargs[key]) for key in kwargs} - # GH43764, making sure any nanoseconds contributions from any kwarg is taken into consideration + # GH43764, making sure any nanoseconds contributions from any kwarg + # is taken into consideration nano = convert_to_timedelta64( ( kwargs.pop('nanoseconds', 0) @@ -1529,10 +1530,6 @@ class Timedelta(_Timedelta): div = other // self return div, other - div * self - # GH40946 - def total_seconds(self): - return self.value / 1e9 - cdef bint is_any_td_scalar(object obj): """ From b8d37e7c41f08c44cea94eca268a2db9ea926431 Mon Sep 17 00:00:00 2001 From: deponovo Date: Fri, 31 Dec 2021 09:35:47 +0100 Subject: [PATCH 6/7] ENH: refactored logic for calculating nanoseconds in the timedelta constructor --- pandas/_libs/tslibs/timedeltas.pyx | 42 +++++++++++++++----------- pandas/tests/tslibs/test_timedeltas.py | 4 +-- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 8fab67eb7db16..09b8b89e306a2 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -180,7 +180,7 @@ cpdef int64_t delta_to_nanoseconds(delta) except? -1: if PyDelta_Check(delta): try: return ( - delta.days * 24 * 60 * 60 * 1_000_000 + delta.days * 24 * 3600 * 1_000_000 + delta.seconds * 1_000_000 + delta.microseconds ) * 1000 @@ -1257,6 +1257,9 @@ class Timedelta(_Timedelta): truncated to nanoseconds. """ + _req_any_kwargs_new = {"weeks", "days", "hours", "minutes", "seconds", + "milliseconds", "microseconds", "nanoseconds"} + def __new__(cls, object value=_no_input, unit=None, **kwargs): cdef _Timedelta td_base @@ -1267,22 +1270,7 @@ class Timedelta(_Timedelta): "(days,seconds....)") kwargs = {key: _to_py_int_float(kwargs[key]) for key in kwargs} - - # GH43764, making sure any nanoseconds contributions from any kwarg - # is taken into consideration - nano = convert_to_timedelta64( - ( - kwargs.pop('nanoseconds', 0) - + kwargs.pop('microseconds', 0) * 1000 - + kwargs.pop('milliseconds', 0) * 1000000 - + kwargs.pop('seconds', 0) * 1000000000 - ), 'ns' - ) - - try: - value = nano + convert_to_timedelta64(timedelta(**kwargs), - 'ns') - except TypeError as e: + if not cls._req_any_kwargs_new.intersection(kwargs): raise ValueError( "cannot construct a Timedelta from the passed arguments, " "allowed keywords are " @@ -1290,6 +1278,26 @@ class Timedelta(_Timedelta): "milliseconds, microseconds, nanoseconds]" ) + # GH43764, making sure any nanoseconds contributions from any kwarg + # is taken into consideration + seconds = int(( + ( + (kwargs.get('days', 0) + kwargs.get('weeks', 0) * 7) * 24 + + kwargs.get('hours', 0) + ) * 3600 + + kwargs.get('minutes', 0) * 60 + + kwargs.get('seconds', 0) + ) * 1_000_000_000 + ) + + value = convert_to_timedelta64( + kwargs.get('nanoseconds', 0) + + int(kwargs.get('microseconds', 0) * 1_000) + + int(kwargs.get('milliseconds', 0) * 1_000_000) + + seconds + , 'ns' + ) + if unit in {'Y', 'y', 'M'}: raise ValueError( "Units 'M', 'Y', and 'y' are no longer supported, as they do not " diff --git a/pandas/tests/tslibs/test_timedeltas.py b/pandas/tests/tslibs/test_timedeltas.py index 0162d09cb5d7c..c72d279580cca 100644 --- a/pandas/tests/tslibs/test_timedeltas.py +++ b/pandas/tests/tslibs/test_timedeltas.py @@ -19,11 +19,11 @@ ( Timedelta(seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 111, - ), # GH43764, GH40946 + ), # GH43764 ( Timedelta(days=1, seconds=1e-9, milliseconds=1e-5, microseconds=1e-1), 24 * 3600e9 + 111, - ), # GH43764, GH40946 + ), # GH43764 (offsets.Nano(125), 125), (1, 1), (np.int64(2), 2), From 88145b5eff4736d6a285b5ce89c2f443e9fb88ea Mon Sep 17 00:00:00 2001 From: deponovo Date: Fri, 31 Dec 2021 17:59:31 +0100 Subject: [PATCH 7/7] PERF: simplified construction of the timedelta DOC: moved the whatsnew entry to the timedelta section and added new comment --- doc/source/whatsnew/v1.4.0.rst | 3 ++- pandas/_libs/tslibs/timedeltas.pyx | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 27b205334ed4d..ec61b22eab318 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -236,7 +236,6 @@ Other enhancements - :meth:`is_list_like` now identifies duck-arrays as list-like unless ``.ndim == 0`` (:issue:`35131`) - :class:`ExtensionDtype` and :class:`ExtensionArray` are now (de)serialized when exporting a :class:`DataFrame` with :meth:`DataFrame.to_json` using ``orient='table'`` (:issue:`20612`, :issue:`44705`). - Add support for `Zstandard `_ compression to :meth:`DataFrame.to_pickle`/:meth:`read_pickle` and friends (:issue:`43925`) -- :class:`Timedelta` now properly taking into account any nanoseconds contribution (:issue:`43764`) - @@ -715,6 +714,8 @@ Timedelta ^^^^^^^^^ - Bug in division of all-``NaT`` :class:`TimeDeltaIndex`, :class:`Series` or :class:`DataFrame` column with object-dtype arraylike of numbers failing to infer the result as timedelta64-dtype (:issue:`39750`) - Bug in floor division of ``timedelta64[ns]`` data with a scalar returning garbage values (:issue:`44466`) +- Bug in :class:`Timedelta` now properly taking into account any nanoseconds contribution of any kwarg (:issue:`43764`) +- Timezones ^^^^^^^^^ diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 09b8b89e306a2..a908eefd41768 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -1278,8 +1278,10 @@ class Timedelta(_Timedelta): "milliseconds, microseconds, nanoseconds]" ) - # GH43764, making sure any nanoseconds contributions from any kwarg - # is taken into consideration + # GH43764, convert any input to nanoseconds first and then + # create the timestamp. This ensures that any potential + # nanosecond contributions from kwargs parsed as floats + # are taken into consideration. seconds = int(( ( (kwargs.get('days', 0) + kwargs.get('weeks', 0) * 7) * 24 @@ -1290,12 +1292,11 @@ class Timedelta(_Timedelta): ) * 1_000_000_000 ) - value = convert_to_timedelta64( - kwargs.get('nanoseconds', 0) + value = np.timedelta64( + int(kwargs.get('nanoseconds', 0)) + int(kwargs.get('microseconds', 0) * 1_000) + int(kwargs.get('milliseconds', 0) * 1_000_000) + seconds - , 'ns' ) if unit in {'Y', 'y', 'M'}: