From 135b2157e848caaf8db680998ccd7851681508ef Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 4 Nov 2020 05:43:46 -0800 Subject: [PATCH 1/4] Backport PR #37613 on branch 1.1.x: BUG: read-only values in cython funcs --- doc/source/whatsnew/v1.1.5.rst | 1 + pandas/_libs/join.pyx | 2 +- pandas/_libs/tslibs/strptime.pyx | 4 ++-- pandas/_libs/tslibs/timedeltas.pyx | 2 +- pandas/core/arrays/datetimelike.py | 3 +-- pandas/tests/test_join.py | 7 ++++++- pandas/tests/tools/test_to_datetime.py | 10 ++++++++++ pandas/tests/tools/test_to_timedelta.py | 10 ++++++++++ 8 files changed, 32 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v1.1.5.rst b/doc/source/whatsnew/v1.1.5.rst index a122154904996..ce988b63b3eea 100644 --- a/doc/source/whatsnew/v1.1.5.rst +++ b/doc/source/whatsnew/v1.1.5.rst @@ -24,6 +24,7 @@ Fixed regressions Bug fixes ~~~~~~~~~ - Bug in metadata propagation for ``groupby`` iterator (:issue:`37343`) +- Bug in :func:`to_datetime` and :func:`to_timedelta` with a read-only array incorrectly raising (:issue:`34857`) - .. --------------------------------------------------------------------------- diff --git a/pandas/_libs/join.pyx b/pandas/_libs/join.pyx index 13c7187923473..1b79d68c13570 100644 --- a/pandas/_libs/join.pyx +++ b/pandas/_libs/join.pyx @@ -268,7 +268,7 @@ ctypedef fused join_t: @cython.wraparound(False) @cython.boundscheck(False) -def left_join_indexer_unique(join_t[:] left, join_t[:] right): +def left_join_indexer_unique(ndarray[join_t] left, ndarray[join_t] right): cdef: Py_ssize_t i, j, nleft, nright ndarray[int64_t] indexer diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index d2690be905a68..bc4632ad028ab 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -12,7 +12,7 @@ from _thread import allocate_lock as _thread_allocate_lock import numpy as np import pytz -from numpy cimport int64_t +from numpy cimport int64_t, ndarray from pandas._libs.tslibs.nattype cimport ( NPY_NAT, @@ -51,7 +51,7 @@ cdef dict _parse_code_table = {'y': 0, 'u': 22} -def array_strptime(object[:] values, object fmt, bint exact=True, errors='raise'): +def array_strptime(ndarray[object] values, object fmt, bint exact=True, errors='raise'): """ Calculates the datetime structs represented by the passed array of strings diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index ee32ed53a908b..083328367407d 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -226,7 +226,7 @@ cdef convert_to_timedelta64(object ts, str unit): @cython.boundscheck(False) @cython.wraparound(False) -def array_to_timedelta64(object[:] values, str unit=None, str errors="raise"): +def array_to_timedelta64(ndarray[object] values, str unit=None, str errors="raise"): """ Convert an ndarray to an array of timedeltas. If errors == 'coerce', coerce non-convertible objects to NaT. Otherwise, raise. diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c6945e2f78b5a..1fe28fd589251 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1334,9 +1334,8 @@ def _addsub_object_array(self, other: np.ndarray, op): result : same class as self """ assert op in [operator.add, operator.sub] - if len(other) == 1: + if len(other) == 1 and self.ndim == 1: # If both 1D then broadcasting is unambiguous - # TODO(EA2D): require self.ndim == other.ndim here return op(self, other[0]) warnings.warn( diff --git a/pandas/tests/test_join.py b/pandas/tests/test_join.py index 129dc275c4d5a..548784bb9b4b6 100644 --- a/pandas/tests/test_join.py +++ b/pandas/tests/test_join.py @@ -43,9 +43,14 @@ def test_outer_join_indexer(self, dtype): tm.assert_numpy_array_equal(rindexer, exp) -def test_left_join_indexer_unique(): +@pytest.mark.parametrize("readonly", [True, False]) +def test_left_join_indexer_unique(readonly): a = np.array([1, 2, 3, 4, 5], dtype=np.int64) b = np.array([2, 2, 3, 4, 4], dtype=np.int64) + if readonly: + # GH#37312, GH#37264 + a.setflags(write=False) + b.setflags(write=False) result = _join.left_join_indexer_unique(b, a) expected = np.array([1, 1, 2, 3, 3], dtype=np.int64) diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index d2049892705ea..4437a11be73e0 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -36,6 +36,16 @@ class TestTimeConversionFormats: + @pytest.mark.parametrize("readonly", [True, False]) + def test_to_datetime_readonly(self, readonly): + # GH#34857 + arr = np.array([], dtype=object) + if readonly: + arr.setflags(write=False) + result = to_datetime(arr) + expected = to_datetime([]) + tm.assert_index_equal(result, expected) + @pytest.mark.parametrize("cache", [True, False]) def test_to_datetime_format(self, cache): values = ["1/1/2000", "1/2/2000", "1/3/2000"] diff --git a/pandas/tests/tools/test_to_timedelta.py b/pandas/tests/tools/test_to_timedelta.py index f68d83f7f4d58..f406253af508f 100644 --- a/pandas/tests/tools/test_to_timedelta.py +++ b/pandas/tests/tools/test_to_timedelta.py @@ -9,6 +9,16 @@ class TestTimedeltas: + @pytest.mark.parametrize("readonly", [True, False]) + def test_to_timedelta_readonly(self, readonly): + # GH#34857 + arr = np.array([], dtype=object) + if readonly: + arr.setflags(write=False) + result = to_timedelta(arr) + expected = to_timedelta([]) + tm.assert_index_equal(result, expected) + def test_to_timedelta(self): result = to_timedelta(["", ""]) From 59eee73e42bb89b7412bc56f7bd763b64628dff1 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 4 Nov 2020 19:52:00 +0000 Subject: [PATCH 2/4] revert change to _addsub_object_array --- pandas/core/arrays/datetimelike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 1fe28fd589251..c6945e2f78b5a 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1334,8 +1334,9 @@ def _addsub_object_array(self, other: np.ndarray, op): result : same class as self """ assert op in [operator.add, operator.sub] - if len(other) == 1 and self.ndim == 1: + if len(other) == 1: # If both 1D then broadcasting is unambiguous + # TODO(EA2D): require self.ndim == other.ndim here return op(self, other[0]) warnings.warn( From 458e0ce754c121fdd7c0792a0f8ed80742e1dafb Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 9 Nov 2020 11:01:34 +0000 Subject: [PATCH 3/4] update condition --- pandas/core/arrays/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c6945e2f78b5a..750d4665ee0c3 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1334,7 +1334,7 @@ def _addsub_object_array(self, other: np.ndarray, op): result : same class as self """ assert op in [operator.add, operator.sub] - if len(other) == 1: + if len(other) == 1 and (self.ndim == 1 or other.shape == self.shape == (1, 1)): # If both 1D then broadcasting is unambiguous # TODO(EA2D): require self.ndim == other.ndim here return op(self, other[0]) From 2a99281dfe0cdf6fa5cf6481fb0c95680061e8d4 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 1 Dec 2020 13:31:28 +0000 Subject: [PATCH 4/4] Revert "update condition" This reverts commit 458e0ce754c121fdd7c0792a0f8ed80742e1dafb. --- pandas/core/arrays/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index e3fb23acb53a1..a9fe95c0892e6 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1334,7 +1334,7 @@ def _addsub_object_array(self, other: np.ndarray, op): result : same class as self """ assert op in [operator.add, operator.sub] - if len(other) == 1 and (self.ndim == 1 or other.shape == self.shape == (1, 1)): + if len(other) == 1: # If both 1D then broadcasting is unambiguous # TODO(EA2D): require self.ndim == other.ndim here return op(self, other[0])