From 52d1040800bbb88e549f49b9d4dfcbe54358012d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 8 Feb 2018 19:58:40 -0800 Subject: [PATCH 1/7] catch more specific error, catch convert_datetime_to_tsobject error correctly --- pandas/_libs/tslib.pyx | 15 +++++++-------- pandas/tests/tslibs/test_tslib.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 pandas/tests/tslibs/test_tslib.py diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 877d7deff6ff4..efa42c33f1638 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -522,11 +522,10 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', seen_datetime = 1 if val.tzinfo is not None: if utc_convert: - _ts = convert_datetime_to_tsobject(val, None) - iresult[i] = _ts.value try: - check_dts_bounds(&_ts.dts) - except ValueError: + _ts = convert_datetime_to_tsobject(val, None) + iresult[i] = _ts.value + except OutOfBoundsDatetime: if is_coerce: iresult[i] = NPY_NAT continue @@ -542,7 +541,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', iresult[i] += val.nanosecond try: check_dts_bounds(&dts) - except ValueError: + except OutOfBoundsDatetime: if is_coerce: iresult[i] = NPY_NAT continue @@ -553,7 +552,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', try: check_dts_bounds(&dts) seen_datetime = 1 - except ValueError: + except OutOfBoundsDatetime: if is_coerce: iresult[i] = NPY_NAT continue @@ -566,7 +565,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', try: iresult[i] = get_datetime64_nanos(val) seen_datetime = 1 - except ValueError: + except OutOfBoundsDatetime: if is_coerce: iresult[i] = NPY_NAT continue @@ -656,7 +655,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', try: _ts = convert_datetime_to_tsobject(py_dt, None) iresult[i] = _ts.value - except ValueError: + except OutOfBoundsDatetime: if is_coerce: iresult[i] = NPY_NAT continue diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py new file mode 100644 index 0000000000000..0eb16601e8526 --- /dev/null +++ b/pandas/tests/tslibs/test_tslib.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from datetime import datetime, timedelta + +import numpy as np + +from pandas._libs.tslib import array_to_datetime, iNaT +from pandas import Timestamp + + +class TestArrayToDatetime(object): + def test_coerce_out_of_bounds_utc(self): + ts = pd.Timestamp('1900-01-01', tz='US/Pacific') + dt = ts.to_pydatetime() - timedelta(days=365*300) # ~1600AD + arr = np.array([dt]) + result = array_to_datetime(arr, utc=True, errors='coerce') + assert (result == iNaT).all() From 49cbe974e179b88d12106e10167881dbae710ef6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 8 Feb 2018 20:25:53 -0800 Subject: [PATCH 2/7] whitespace fixup --- pandas/tests/tslibs/test_tslib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py index 0eb16601e8526..37b8f3e19c8f2 100644 --- a/pandas/tests/tslibs/test_tslib.py +++ b/pandas/tests/tslibs/test_tslib.py @@ -10,7 +10,7 @@ class TestArrayToDatetime(object): def test_coerce_out_of_bounds_utc(self): ts = pd.Timestamp('1900-01-01', tz='US/Pacific') - dt = ts.to_pydatetime() - timedelta(days=365*300) # ~1600AD + dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD arr = np.array([dt]) result = array_to_datetime(arr, utc=True, errors='coerce') assert (result == iNaT).all() From 68a11b3cbfdaff9837675b1509a8e49f0eda0a1e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 8 Feb 2018 22:10:36 -0800 Subject: [PATCH 3/7] fixup namerror --- pandas/tests/tslibs/test_tslib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py index 37b8f3e19c8f2..c34f6127fa85c 100644 --- a/pandas/tests/tslibs/test_tslib.py +++ b/pandas/tests/tslibs/test_tslib.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from datetime import datetime, timedelta +from datetime import timedelta import numpy as np @@ -9,7 +9,7 @@ class TestArrayToDatetime(object): def test_coerce_out_of_bounds_utc(self): - ts = pd.Timestamp('1900-01-01', tz='US/Pacific') + ts = Timestamp('1900-01-01', tz='US/Pacific') dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD arr = np.array([dt]) result = array_to_datetime(arr, utc=True, errors='coerce') From 4fdef0382aa5b4b4f658243dda40ffb6ad05e7f0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 9 Feb 2018 08:00:32 -0800 Subject: [PATCH 4/7] fix assertion --- pandas/tests/tslibs/test_tslib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py index c34f6127fa85c..e9bd33498dbde 100644 --- a/pandas/tests/tslibs/test_tslib.py +++ b/pandas/tests/tslibs/test_tslib.py @@ -6,6 +6,8 @@ from pandas._libs.tslib import array_to_datetime, iNaT from pandas import Timestamp +import pandas.util.testing as tm + class TestArrayToDatetime(object): def test_coerce_out_of_bounds_utc(self): @@ -13,4 +15,5 @@ def test_coerce_out_of_bounds_utc(self): dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD arr = np.array([dt]) result = array_to_datetime(arr, utc=True, errors='coerce') - assert (result == iNaT).all() + expected = np.array(['NaT'], dtype='datetime64[ns]') + tm.assert_numpy_array_equal(result, expected) From c45d2827041460b4b35edcf8f01e9ca26990aec5 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 9 Feb 2018 09:40:44 -0800 Subject: [PATCH 5/7] remove unused import --- pandas/tests/tslibs/test_tslib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py index e9bd33498dbde..c3c66d3f7177a 100644 --- a/pandas/tests/tslibs/test_tslib.py +++ b/pandas/tests/tslibs/test_tslib.py @@ -3,7 +3,7 @@ import numpy as np -from pandas._libs.tslib import array_to_datetime, iNaT +from pandas._libs.tslib import array_to_datetime from pandas import Timestamp import pandas.util.testing as tm From 4b1b9f8a8cdd119da80a2f10731f8f8eb162c1a8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 10 Feb 2018 10:10:01 -0800 Subject: [PATCH 6/7] move test --- pandas/tests/indexes/datetimes/test_tools.py | 10 +++++++++- pandas/tests/tslibs/test_tslib.py | 19 ------------------- 2 files changed, 9 insertions(+), 20 deletions(-) delete mode 100644 pandas/tests/tslibs/test_tslib.py diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index b95ae07052ecb..c4635de5a6197 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -8,7 +8,7 @@ import dateutil import numpy as np from dateutil.parser import parse -from datetime import datetime, date, time +from datetime import datetime, date, time, timedelta from distutils.version import LooseVersion import pandas as pd @@ -1491,6 +1491,14 @@ def test_parsers_iso8601(self): class TestArrayToDatetime(object): + def test_coerce_out_of_bounds_utc(self): + ts = Timestamp('1900-01-01', tz='US/Pacific') + dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD + arr = np.array([dt]) + result = tslib.array_to_datetime(arr, utc=True, errors='coerce') + expected = np.array(['NaT'], dtype='datetime64[ns]') + tm.assert_numpy_array_equal(result, expected) + def test_parsing_valid_dates(self): arr = np.array(['01-01-2013', '01-02-2013'], dtype=object) tm.assert_numpy_array_equal( diff --git a/pandas/tests/tslibs/test_tslib.py b/pandas/tests/tslibs/test_tslib.py deleted file mode 100644 index c3c66d3f7177a..0000000000000 --- a/pandas/tests/tslibs/test_tslib.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import timedelta - -import numpy as np - -from pandas._libs.tslib import array_to_datetime -from pandas import Timestamp - -import pandas.util.testing as tm - - -class TestArrayToDatetime(object): - def test_coerce_out_of_bounds_utc(self): - ts = Timestamp('1900-01-01', tz='US/Pacific') - dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD - arr = np.array([dt]) - result = array_to_datetime(arr, utc=True, errors='coerce') - expected = np.array(['NaT'], dtype='datetime64[ns]') - tm.assert_numpy_array_equal(result, expected) From 9999e2dd0ef7d2fad9b3c5c718c76acbed835c1a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 10 Feb 2018 10:14:56 -0800 Subject: [PATCH 7/7] Whatsnew note --- doc/source/whatsnew/v0.23.0.txt | 3 ++- pandas/tests/indexes/datetimes/test_tools.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 6f48d9a6c63c9..41d6520b3d91a 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -703,7 +703,7 @@ Datetimelike - Bug in :class:`Series` floor-division where operating on a scalar ``timedelta`` raises an exception (:issue:`18846`) - Bug in :class:`Series`` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` had results cast to ``dtype='int64'`` (:issue:`17250`) - Bug in :class:`TimedeltaIndex` where division by a ``Series`` would return a ``TimedeltaIndex`` instead of a ``Series`` (issue:`19042`) -- Bug in :class:`Series` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` could return a ``Series`` with an incorrect name (issue:`19043`) +- Bug in :class:`Series` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` could return a ``Series`` with an incorrect name (:issue:`19043`) - Bug in :class:`DatetimeIndex` where the repr was not showing high-precision time values at the end of a day (e.g., 23:59:59.999999999) (:issue:`19030`) - Bug where dividing a scalar timedelta-like object with :class:`TimedeltaIndex` performed the reciprocal operation (:issue:`19125`) - Bug in ``.astype()`` to non-ns timedelta units would hold the incorrect dtype (:issue:`19176`, :issue:`19223`, :issue:`12425`) @@ -713,6 +713,7 @@ Datetimelike - Bug in comparison of :class:`DatetimeIndex` against ``None`` or ``datetime.date`` objects raising ``TypeError`` for ``==`` and ``!=`` comparisons instead of all-``False`` and all-``True``, respectively (:issue:`19301`) - Bug in :class:`Timestamp` and :func:`to_datetime` where a string representing a barely out-of-bounds timestamp would be incorrectly rounded down instead of raising ``OutOfBoundsDatetime`` (:issue:`19382`) - Bug in :func:`Timestamp.floor` :func:`DatetimeIndex.floor` where time stamps far in the future and past were not rounded correctly (:issue:`19206`) +- Bug in :func:`to_datetime` where passing an out-of-bounds datetime with ``errors='coerce'`` and ``utc=True`` would raise ``OutOfBoundsDatetime`` instead of parsing to ``NaT`` (:issue:`19612`) - Timezones diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index c4635de5a6197..1bdf5c07e1cd0 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -1492,6 +1492,7 @@ def test_parsers_iso8601(self): class TestArrayToDatetime(object): def test_coerce_out_of_bounds_utc(self): + # GH#19612 ts = Timestamp('1900-01-01', tz='US/Pacific') dt = ts.to_pydatetime() - timedelta(days=365 * 300) # ~1600AD arr = np.array([dt])