From 61ad5104174ff5decdc3316abce6d7c2f60aeb1a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 5 Nov 2018 17:07:43 -0800 Subject: [PATCH 01/12] BUG: fix and test offset comparison with non-offsets --- pandas/_libs/tslibs/offsets.pyx | 7 +++-- pandas/tests/tseries/offsets/test_offsets.py | 12 +++++++++ pandas/tests/tseries/offsets/test_ticks.py | 21 +++++++++++++++ pandas/tseries/offsets.py | 27 +++++++++++++++----- 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 78e1269aa5363..d739a60380805 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -306,8 +306,11 @@ class _BaseOffset(object): def __eq__(self, other): if is_string_object(other): - other = to_offset(other) - + try: + other = to_offset(other) + except ValueError: + # e.g. "infer" + return False try: return self._params == other._params except AttributeError: diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index cbd3e0903b713..7c6f0b59192dc 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -187,6 +187,18 @@ def testMult2(self): assert self.d + (-5 * self._offset(-10)) == self.d + self._offset(50) assert self.d + (-3 * self._offset(-2)) == self.d + self._offset(6) + def test_compare_str(self): + # comparing to strings that cannot be cast to DateOffsets should + # not raise for __eq__ or __ne__ + if self._offset is None: + return + off = self._get_offset(self._offset) + + assert not off == "infer" + assert off != "foo" + # Note: inequalities are only implemented for Tick subclasses; + # tests for this are in test_ticks + class TestCommon(Base): # exected value created by Base._get_offset diff --git a/pandas/tests/tseries/offsets/test_ticks.py b/pandas/tests/tseries/offsets/test_ticks.py index 369c0971f1e9a..d83266c2b2a92 100644 --- a/pandas/tests/tseries/offsets/test_ticks.py +++ b/pandas/tests/tseries/offsets/test_ticks.py @@ -267,3 +267,24 @@ def test_compare_ticks(cls): assert cls(4) > three assert cls(3) == cls(3) assert cls(3) != cls(4) + + +@pytest.mark.parametrize('cls', tick_classes) +def test_compare_ticks_to_strs(cls): + off = cls(19) + + # These tests should work with any strings, but we particularly are + # interested in "infer" as that comparison is convenient to make in + # Datetime/Timedelta Array/Index constructors + assert not off == "infer" + assert not "foo" == off + + for left, right in [("infer", off), (off, "infer")]: + with pytest.raises(TypeError): + left < right + with pytest.raises(TypeError): + left <= right + with pytest.raises(TypeError): + left > right + with pytest.raises(TypeError): + left >= right diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 6fb562e301ac2..133b03dee375c 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2199,9 +2199,18 @@ def apply_index(self, i): def _tick_comp(op): + assert op not in [operator.eq, operator.ne] + def f(self, other): - return op(self.delta, other.delta) + try: + return op(self.delta, other.delta) + except AttributeError: + # comparing with a non-Tick object + raise TypeError("Invalid comparison between {cls} and {typ}" + .format(cls=type(self).__name__, + typ=type(other).__name__)) + f.__name__ = '__{opname}__'.format(opname=op.__name__) return f @@ -2220,8 +2229,6 @@ def __init__(self, n=1, normalize=False): __ge__ = _tick_comp(operator.ge) __lt__ = _tick_comp(operator.lt) __le__ = _tick_comp(operator.le) - __eq__ = _tick_comp(operator.eq) - __ne__ = _tick_comp(operator.ne) def __add__(self, other): if isinstance(other, Tick): @@ -2242,8 +2249,11 @@ def __add__(self, other): def __eq__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset - - other = to_offset(other) + try: + other = to_offset(other) + except ValueError: + # e.g. "infer" + return False if isinstance(other, Tick): return self.delta == other.delta @@ -2258,8 +2268,11 @@ def __hash__(self): def __ne__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset - - other = to_offset(other) + try: + other = to_offset(other) + except ValueError: + # e.g. "infer" + return True if isinstance(other, Tick): return self.delta != other.delta From 937541e2ed3be2284d743407f8e482009ebac5d8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 5 Nov 2018 17:09:15 -0800 Subject: [PATCH 02/12] whatsnew note --- doc/source/whatsnew/v0.24.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 6ace245a4bae1..f955e9fc0adf4 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1162,6 +1162,7 @@ Offsets - Bug in :class:`FY5253` where date offsets could incorrectly raise an ``AssertionError`` in arithmetic operatons (:issue:`14774`) - Bug in :class:`DateOffset` where keyword arguments ``week`` and ``milliseconds`` were accepted and ignored. Passing these will now raise ``ValueError`` (:issue:`19398`) - Bug in adding :class:`DateOffset` with :class:`DataFrame` or :class:`PeriodIndex` incorrectly raising ``TypeError`` (:issue:`23215`) +- Bug in comparing :class:`DateOffset` objects with non-DateOffset objects, particularly strings, raising ``ValueError`` instead of returning ``False`` for equality checks and ``True`` for not-equal checks (:issue:`????`) Numeric ^^^^^^^ From ca9e8af967817b7a922868ae7405d0eb6ed8fba2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 5 Nov 2018 17:50:56 -0800 Subject: [PATCH 03/12] Fix and test casting tz-aware datetimeindex to object-dtype ndarray or Index --- doc/source/whatsnew/v0.24.0.txt | 2 ++ pandas/core/arrays/datetimes.py | 10 ++++++++++ pandas/core/indexes/base.py | 10 +++++++--- pandas/tests/arrays/test_datetimelike.py | 23 +++++++++++++++++++++++ pandas/tests/indexes/test_base.py | 12 ++++++++++-- 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index f955e9fc0adf4..8943bb1ce3a74 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1357,3 +1357,5 @@ Other - :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax`` (:issue:`21548` and :issue:`21526`). ``NaN`` values are also handled properly. - Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`) - Bug in :meth:`DataFrame.combine_first` in which column types were unexpectedly converted to float (:issue:`20699`) +- Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`????`) +- Bug in :class:`DatetimeIndex` where calling `np.array(dtindex, dtype=object)` would incorrectly return an array of ``long`` objects (:issue:`????`) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 0258e1e6e5973..00ae2ccbdc0d2 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -374,6 +374,16 @@ def _resolution(self): # ---------------------------------------------------------------- # Array-like Methods + def __array__(self, dtype=None): + if is_object_dtype(dtype): + return np.array(list(self), dtype=object) + elif dtype == 'i8': + return self.asi8 + + # TODO: warn that dtype is not used? + # warn that conversion may be lossy? + return self._data.view(np.ndarray) # follow Index.__array__ + def __iter__(self): """ Return an iterator over the boxed values diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index ae64179b36485..fbdb25d67fac0 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -301,11 +301,15 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, (dtype is not None and is_datetime64_any_dtype(dtype)) or 'tz' in kwargs): from pandas import DatetimeIndex - result = DatetimeIndex(data, copy=copy, name=name, - dtype=dtype, **kwargs) + if dtype is not None and is_dtype_equal(_o_dtype, dtype): - return Index(result.to_pydatetime(), dtype=_o_dtype) + # if `data` is already a TZaware DatetimeIndex, then passing + # dtype=object to the constructor will raise spuriously + result = DatetimeIndex(data, copy=copy, name=name, **kwargs) + return Index(list(result), dtype=_o_dtype) else: + result = DatetimeIndex(data, copy=copy, name=name, + dtype=dtype, **kwargs) return result elif (is_timedelta64_dtype(data) or diff --git a/pandas/tests/arrays/test_datetimelike.py b/pandas/tests/arrays/test_datetimelike.py index 3fd03a351de7c..7caefda48b1fa 100644 --- a/pandas/tests/arrays/test_datetimelike.py +++ b/pandas/tests/arrays/test_datetimelike.py @@ -57,6 +57,29 @@ def timedelta_index(request): class TestDatetimeArray(object): + def test_array_object_dtype(self, tz_naive_fixture): + tz = tz_naive_fixture + dti = pd.date_range('2016-01-01', periods=3, tz=tz) + arr = DatetimeArrayMixin(dti) + + expected = np.array(list(dti)) + + result = np.array(arr, dtype=object) + tm.assert_numpy_array_equal(result, expected) + + # also test the DatetimeIndex method while we're at it + result = np.array(dti, dtype=object) + tm.assert_numpy_array_equal(result, expected) + + def test_array(self, tz_naive_fixture): + tz = tz_naive_fixture + dti = pd.date_range('2016-01-01', periods=3, tz=tz) + arr = DatetimeArrayMixin(dti) + + expected = dti.asi8.view('M8[ns]') + result = np.array(arr) + tm.assert_numpy_array_equal(result, expected) + def test_from_dti(self, tz_naive_fixture): tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=3, tz=tz) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index daebc6e95aac4..592df07a74f49 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -132,7 +132,7 @@ def test_construction_list_tuples_nan(self, na_value, vtype): @pytest.mark.parametrize("cast_as_obj", [True, False]) @pytest.mark.parametrize("index", [ pd.date_range('2015-01-01 10:00', freq='D', periods=3, - tz='US/Eastern'), # DTI with tz + tz='US/Eastern', name='Green Eggs & Ham'), # DTI with tz pd.date_range('2015-01-01 10:00', freq='D', periods=3), # DTI no tz pd.timedelta_range('1 days', freq='D', periods=3), # td pd.period_range('2015-01-01', freq='D', periods=3) # period @@ -145,8 +145,16 @@ def test_constructor_from_index_dtlike(self, cast_as_obj, index): tm.assert_index_equal(result, index) - if isinstance(index, pd.DatetimeIndex) and hasattr(index, 'tz'): + if isinstance(index, pd.DatetimeIndex): assert result.tz == index.tz + if cast_as_obj: + # GH#???? check that Index(dti, dtype=object) does not + # incorrectly raise ValueError, and that nanoseconds are not + # dropped + index += pd.Timedelta(nanoseconds=50) + result = pd.Index(index, dtype=object) + assert result.dtype == np.object_ + assert list(result) == list(index) @pytest.mark.parametrize("index,has_tz", [ (pd.date_range('2015-01-01 10:00', freq='D', periods=3, From 9d0fbd7ef4a123f141ef1fac98afa011ad9b56d3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 5 Nov 2018 17:57:33 -0800 Subject: [PATCH 04/12] add GH references --- doc/source/whatsnew/v0.24.0.txt | 6 +++--- pandas/core/indexes/base.py | 4 ++-- pandas/tests/arrays/test_datetimelike.py | 2 ++ pandas/tests/indexes/test_base.py | 2 +- pandas/tests/tseries/offsets/test_offsets.py | 1 + pandas/tests/tseries/offsets/test_ticks.py | 1 + 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 8943bb1ce3a74..3586a617a0f35 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1162,7 +1162,7 @@ Offsets - Bug in :class:`FY5253` where date offsets could incorrectly raise an ``AssertionError`` in arithmetic operatons (:issue:`14774`) - Bug in :class:`DateOffset` where keyword arguments ``week`` and ``milliseconds`` were accepted and ignored. Passing these will now raise ``ValueError`` (:issue:`19398`) - Bug in adding :class:`DateOffset` with :class:`DataFrame` or :class:`PeriodIndex` incorrectly raising ``TypeError`` (:issue:`23215`) -- Bug in comparing :class:`DateOffset` objects with non-DateOffset objects, particularly strings, raising ``ValueError`` instead of returning ``False`` for equality checks and ``True`` for not-equal checks (:issue:`????`) +- Bug in comparing :class:`DateOffset` objects with non-DateOffset objects, particularly strings, raising ``ValueError`` instead of returning ``False`` for equality checks and ``True`` for not-equal checks (:issue:`23524`) Numeric ^^^^^^^ @@ -1357,5 +1357,5 @@ Other - :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax`` (:issue:`21548` and :issue:`21526`). ``NaN`` values are also handled properly. - Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`) - Bug in :meth:`DataFrame.combine_first` in which column types were unexpectedly converted to float (:issue:`20699`) -- Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`????`) -- Bug in :class:`DatetimeIndex` where calling `np.array(dtindex, dtype=object)` would incorrectly return an array of ``long`` objects (:issue:`????`) +- Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) +- Bug in :class:`DatetimeIndex` where calling `np.array(dtindex, dtype=object)` would incorrectly return an array of ``long`` objects (:issue:`23524`) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index fbdb25d67fac0..73f699c89dc75 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -303,8 +303,8 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, from pandas import DatetimeIndex if dtype is not None and is_dtype_equal(_o_dtype, dtype): - # if `data` is already a TZaware DatetimeIndex, then passing - # dtype=object to the constructor will raise spuriously + # GH#23524 if `data` is already a TZaware DatetimeIndex, + # then passing dtype=object to the constructor would raise result = DatetimeIndex(data, copy=copy, name=name, **kwargs) return Index(list(result), dtype=_o_dtype) else: diff --git a/pandas/tests/arrays/test_datetimelike.py b/pandas/tests/arrays/test_datetimelike.py index 7caefda48b1fa..377531c1c748d 100644 --- a/pandas/tests/arrays/test_datetimelike.py +++ b/pandas/tests/arrays/test_datetimelike.py @@ -58,6 +58,7 @@ def timedelta_index(request): class TestDatetimeArray(object): def test_array_object_dtype(self, tz_naive_fixture): + # GH#23524 tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=3, tz=tz) arr = DatetimeArrayMixin(dti) @@ -72,6 +73,7 @@ def test_array_object_dtype(self, tz_naive_fixture): tm.assert_numpy_array_equal(result, expected) def test_array(self, tz_naive_fixture): + # GH#23524 tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=3, tz=tz) arr = DatetimeArrayMixin(dti) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 592df07a74f49..726db0cdee4fe 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -148,7 +148,7 @@ def test_constructor_from_index_dtlike(self, cast_as_obj, index): if isinstance(index, pd.DatetimeIndex): assert result.tz == index.tz if cast_as_obj: - # GH#???? check that Index(dti, dtype=object) does not + # GH#23524 check that Index(dti, dtype=object) does not # incorrectly raise ValueError, and that nanoseconds are not # dropped index += pd.Timedelta(nanoseconds=50) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 7c6f0b59192dc..d68dd65c9841b 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -188,6 +188,7 @@ def testMult2(self): assert self.d + (-3 * self._offset(-2)) == self.d + self._offset(6) def test_compare_str(self): + # GH#23524 # comparing to strings that cannot be cast to DateOffsets should # not raise for __eq__ or __ne__ if self._offset is None: diff --git a/pandas/tests/tseries/offsets/test_ticks.py b/pandas/tests/tseries/offsets/test_ticks.py index d83266c2b2a92..128010fe6d32c 100644 --- a/pandas/tests/tseries/offsets/test_ticks.py +++ b/pandas/tests/tseries/offsets/test_ticks.py @@ -271,6 +271,7 @@ def test_compare_ticks(cls): @pytest.mark.parametrize('cls', tick_classes) def test_compare_ticks_to_strs(cls): + # GH#23524 off = cls(19) # These tests should work with any strings, but we particularly are From dbf145f0ac3941d5d12ddef8a431151049100bef Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 5 Nov 2018 19:09:00 -0800 Subject: [PATCH 05/12] Clarify comment --- pandas/core/indexes/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 73f699c89dc75..b2a68770eb554 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -303,8 +303,10 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, from pandas import DatetimeIndex if dtype is not None and is_dtype_equal(_o_dtype, dtype): - # GH#23524 if `data` is already a TZaware DatetimeIndex, - # then passing dtype=object to the constructor would raise + # GH#23524 passing `dtype=object` to DatetimeIndex is invalid, + # will raise in the where `data` is already tz-aware. So + # we leave it out of this step and cast to object-dtype after + # the DatetimeIndex construction. result = DatetimeIndex(data, copy=copy, name=name, **kwargs) return Index(list(result), dtype=_o_dtype) else: From 8853b6a43448945fce0e4a555c15087b1a635b48 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 6 Nov 2018 08:14:30 -0800 Subject: [PATCH 06/12] test for np.array(arr, dtype=np.int64) --- pandas/core/arrays/datetimes.py | 3 ++- pandas/tests/arrays/test_datetimelike.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 0c9455388e9c5..e65d0744c4138 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -19,6 +19,7 @@ from pandas.core.dtypes.common import ( _NS_DTYPE, is_object_dtype, + is_int64_dtype, is_datetime64tz_dtype, is_datetime64_dtype, ensure_int64) @@ -379,7 +380,7 @@ def _resolution(self): def __array__(self, dtype=None): if is_object_dtype(dtype): return np.array(list(self), dtype=object) - elif dtype == 'i8': + elif is_int64_dtype(dtype): return self.asi8 # TODO: warn that dtype is not used? diff --git a/pandas/tests/arrays/test_datetimelike.py b/pandas/tests/arrays/test_datetimelike.py index 377531c1c748d..638f88f44f9ac 100644 --- a/pandas/tests/arrays/test_datetimelike.py +++ b/pandas/tests/arrays/test_datetimelike.py @@ -82,6 +82,19 @@ def test_array(self, tz_naive_fixture): result = np.array(arr) tm.assert_numpy_array_equal(result, expected) + def test_array_i8_dtype(self, tz_naive_fixture): + # GH#23524 + tz = tz_naive_fixture + dti = pd.date_range('2016-01-01', periods=3, tz=tz) + arr = DatetimeArrayMixin(dti) + + expected = dti.asi8 + result = np.array(arr, dtype='i8') + tm.assert_numpy_array_equal(result, expected) + + result = np.array(arr, dtype=np.int64) + tm.assert_numpy_array_equal(result, expected) + def test_from_dti(self, tz_naive_fixture): tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=3, tz=tz) From 9add480c09be6edbe1ea1b1d4f5a9f02ffd324f0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 6 Nov 2018 08:34:46 -0800 Subject: [PATCH 07/12] release note for fixing dropping of nanoseconds --- doc/source/whatsnew/v0.24.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index d6f482a904807..a7c16856fee71 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1367,3 +1367,4 @@ Other - Bug in :meth:`DataFrame.combine_first` in which column types were unexpectedly converted to float (:issue:`20699`) - Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) - Bug in :class:`DatetimeIndex` where calling `np.array(dtindex, dtype=object)` would incorrectly return an array of ``long`` objects (:issue:`23524`) +- Bug in :class:`Index` where calling `np.array(dtindex, dtype=object)` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) From cfd8e717e90effbbd8d54f1ac125c0a0e843d17c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 6 Nov 2018 17:01:54 -0800 Subject: [PATCH 08/12] test for copy=False being respected --- pandas/tests/arrays/test_datetimelike.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pandas/tests/arrays/test_datetimelike.py b/pandas/tests/arrays/test_datetimelike.py index 638f88f44f9ac..5ba99a48e34ad 100644 --- a/pandas/tests/arrays/test_datetimelike.py +++ b/pandas/tests/arrays/test_datetimelike.py @@ -82,6 +82,11 @@ def test_array(self, tz_naive_fixture): result = np.array(arr) tm.assert_numpy_array_equal(result, expected) + # check that we are not making copies when setting copy=False + result = np.array(arr, copy=False) + assert result.base is expected.base + assert result.base is not None + def test_array_i8_dtype(self, tz_naive_fixture): # GH#23524 tz = tz_naive_fixture @@ -95,6 +100,11 @@ def test_array_i8_dtype(self, tz_naive_fixture): result = np.array(arr, dtype=np.int64) tm.assert_numpy_array_equal(result, expected) + # check that we are not making copies when setting copy=False + result = np.array(arr, dtype='i8', copy=False) + assert result.base is expected.base + assert result.base is not None + def test_from_dti(self, tz_naive_fixture): tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=3, tz=tz) From b4ecfbb1d3180acc223a8bb1e0553ec0579e378e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 7 Nov 2018 08:02:13 -0800 Subject: [PATCH 09/12] fix missing backticks --- doc/source/whatsnew/v0.24.0.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 497d13d4c098d..1d026eb2c4f00 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1372,5 +1372,5 @@ Other - Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`) - Bug in :meth:`DataFrame.combine_first` in which column types were unexpectedly converted to float (:issue:`20699`) - Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) -- Bug in :class:`DatetimeIndex` where calling `np.array(dtindex, dtype=object)` would incorrectly return an array of ``long`` objects (:issue:`23524`) -- Bug in :class:`Index` where calling `np.array(dtindex, dtype=object)` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) +- Bug in :class:`DatetimeIndex` where calling ``np.array(dtindex, dtype=object)`` would incorrectly return an array of ``long`` objects (:issue:`23524`) +- Bug in :class:`Index` where calling ``np.array(dtindex, dtype=object)`` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) From 756574f46d5ee2c573df8d28ac1540f49bfb18d8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 7 Nov 2018 08:05:45 -0800 Subject: [PATCH 10/12] comments --- pandas/_libs/tslibs/offsets.pyx | 2 ++ pandas/tseries/offsets.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 34e4faa5fd274..74442708a0f42 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -307,6 +307,8 @@ class _BaseOffset(object): def __eq__(self, other): if is_string_object(other): try: + # GH#23524 if to_offset fails, we are dealing with an + # incomparable type so == is False and != is True other = to_offset(other) except ValueError: # e.g. "infer" diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 133b03dee375c..83bd7aeb6ee19 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2250,6 +2250,8 @@ def __eq__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset try: + # GH#23524 if to_offset fails, we are dealing with an + # incomparable type so == is False and != is True other = to_offset(other) except ValueError: # e.g. "infer" @@ -2269,6 +2271,8 @@ def __ne__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset try: + # GH#23524 if to_offset fails, we are dealing with an + # incomparable type so == is False and != is True other = to_offset(other) except ValueError: # e.g. "infer" From bf1677addae30c468ab316291fc7637b8273c83a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 8 Nov 2018 06:56:29 -0800 Subject: [PATCH 11/12] move whatsnew note, do astype differently --- doc/source/whatsnew/v0.24.0.txt | 6 +++--- pandas/core/indexes/base.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index a2ed86afa2ffa..7d831092f1549 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -1124,6 +1124,9 @@ Datetimelike - Bug in :class:`PeriodIndex` with attribute ``freq.n`` greater than 1 where adding a :class:`DateOffset` object would return incorrect results (:issue:`23215`) - Bug in :class:`Series` that interpreted string indices as lists of characters when setting datetimelike values (:issue:`23451`) - Bug in :class:`Timestamp` constructor which would drop the frequency of an input :class:`Timestamp` (:issue:`22311`) +- Bug in :class:`DatetimeIndex` where calling ``np.array(dtindex, dtype=object)`` would incorrectly return an array of ``long`` objects (:issue:`23524`) +- Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) +- Bug in :class:`Index` where calling ``np.array(dtindex, dtype=object)`` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) Timedelta ^^^^^^^^^ @@ -1373,6 +1376,3 @@ Other - :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax`` (:issue:`21548` and :issue:`21526`). ``NaN`` values are also handled properly. - Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`) - Bug in :meth:`DataFrame.combine_first` in which column types were unexpectedly converted to float (:issue:`20699`) -- Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) -- Bug in :class:`DatetimeIndex` where calling ``np.array(dtindex, dtype=object)`` would incorrectly return an array of ``long`` objects (:issue:`23524`) -- Bug in :class:`Index` where calling ``np.array(dtindex, dtype=object)`` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 57d260ec553c4..60d38b766ee4c 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -307,8 +307,10 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, # will raise in the where `data` is already tz-aware. So # we leave it out of this step and cast to object-dtype after # the DatetimeIndex construction. - result = DatetimeIndex(data, copy=copy, name=name, **kwargs) - return Index(list(result), dtype=_o_dtype) + # Note we can pass copy=False because the .astype below + # will always make a copy + result = DatetimeIndex(data, copy=False, name=name, **kwargs) + return result.astype(object) else: result = DatetimeIndex(data, copy=copy, name=name, dtype=dtype, **kwargs) From 0a5dbed0f27f7610211881862844b622c70ff631 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 9 Nov 2018 06:01:46 -0800 Subject: [PATCH 12/12] remove comment --- pandas/core/arrays/datetimes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index e65d0744c4138..3a2c21b4ddf95 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -383,8 +383,7 @@ def __array__(self, dtype=None): elif is_int64_dtype(dtype): return self.asi8 - # TODO: warn that dtype is not used? - # warn that conversion may be lossy? + # TODO: warn that conversion may be lossy? return self._data.view(np.ndarray) # follow Index.__array__ def __iter__(self):