From c613f2ea837f4277bd722305f4657be8a1a10fe3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 6 Jan 2020 13:24:24 -0800 Subject: [PATCH 1/7] BUG: TDI.insert with empty TDI raising IndexError --- pandas/core/indexes/datetimes.py | 11 +++++------ pandas/core/indexes/period.py | 2 +- pandas/core/indexes/timedeltas.py | 14 +++++++++----- pandas/tests/indexes/datetimes/test_indexing.py | 4 ++-- pandas/tests/indexes/timedeltas/test_indexing.py | 9 +++++++++ 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 40d3823c9700b..d7e9b7076e55c 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -907,7 +907,7 @@ def inferred_type(self) -> str: # sure we can't have ambiguous indexing return "datetime64" - def insert(self, loc, item): + def insert(self, loc: int, item): """ Make new Index inserting new item at location @@ -928,10 +928,9 @@ def insert(self, loc, item): freq = None - if isinstance(item, (datetime, np.datetime64)): - self._assert_can_do_op(item) - if not self._has_same_tz(item) and not isna(item): - raise ValueError("Passed item and index have different timezone") + if isinstance(item, self._data._recognized_scalars) or item is NaT: + item = self._data._scalar_type(item) + self._data._check_compatible_with(item, setitem=True) # check freq can be preserved on edge cases if self.size and self.freq is not None: @@ -941,7 +940,7 @@ def insert(self, loc, item): freq = self.freq elif (loc == len(self)) and item - self.freq == self[-1]: freq = self.freq - item = _to_M8(item, tz=self.tz) + item = item.asm8 try: new_dates = np.concatenate( diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index 6ba778bc83dd1..fb3b9f2f38718 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -732,7 +732,7 @@ def _convert_tolerance(self, tolerance, target): raise ValueError("list-like tolerance size must match target index size") return self._maybe_convert_timedelta(tolerance) - def insert(self, loc, item): + def insert(self, loc: int, item): if not isinstance(item, Period) or self.freq != item.freq: return self.astype(object).insert(loc, item) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index ee6e5b984ae7b..bf04d07d62411 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -381,7 +381,7 @@ def is_type_compatible(self, typ) -> bool: def inferred_type(self) -> str: return "timedelta64" - def insert(self, loc, item): + def insert(self, loc: int, item): """ Make new Index inserting new item at location @@ -408,15 +408,19 @@ def insert(self, loc, item): item = self._na_value freq = None - if isinstance(item, Timedelta) or (is_scalar(item) and isna(item)): + if isinstance(item, self._data._recognized_scalars) or item is NaT: + item = self._data._scalar_type(item) + self._data._check_compatible_with(item, setitem=True) # check freq can be preserved on edge cases - if self.freq is not None: - if (loc == 0 or loc == -len(self)) and item + self.freq == self[0]: + if self.size and self.freq is not None: + if item is NaT: + pass + elif (loc == 0 or loc == -len(self)) and item + self.freq == self[0]: freq = self.freq elif (loc == len(self)) and item - self.freq == self[-1]: freq = self.freq - item = Timedelta(item).asm8.view(_TD_DTYPE) + item = item.asm8 try: new_tds = np.concatenate( diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index ef0d2cd2e48cc..ba926696a4761 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -403,9 +403,9 @@ def test_insert(self): # see gh-7299 idx = date_range("1/1/2000", periods=3, freq="D", tz="Asia/Tokyo", name="idx") - with pytest.raises(ValueError): + with pytest.raises(TypeError): idx.insert(3, pd.Timestamp("2000-01-04")) - with pytest.raises(ValueError): + with pytest.raises(TypeError): idx.insert(3, datetime(2000, 1, 4)) with pytest.raises(ValueError): idx.insert(3, pd.Timestamp("2000-01-04", tz="US/Eastern")) diff --git a/pandas/tests/indexes/timedeltas/test_indexing.py b/pandas/tests/indexes/timedeltas/test_indexing.py index 0114dfef548de..fdb4b3222ea07 100644 --- a/pandas/tests/indexes/timedeltas/test_indexing.py +++ b/pandas/tests/indexes/timedeltas/test_indexing.py @@ -161,6 +161,15 @@ def test_take_fill_value(self): class TestTimedeltaIndex: + def test_insert_empty(self): + # Corner case inserting with length zero doesnt raise IndexError + idx = timedelta_range("1 Day", periods=3) + td = idx[0] + + idx[:0].insert(0, td) + idx[:0].insert(1, td) + idx[:0].insert(-1, td) + def test_insert(self): idx = TimedeltaIndex(["4day", "1day", "2day"], name="idx") From c4a1b3a50e53191f496fcf719128f532a90e0cd5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 6 Jan 2020 13:49:18 -0800 Subject: [PATCH 2/7] mypy fixup --- pandas/core/indexes/datetimelike.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 34560065525dd..81772a9ce3e3d 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -2,7 +2,7 @@ Base and utility classes for tseries type pandas objects. """ import operator -from typing import List, Optional, Set +from typing import List, Optional, Set, Union import numpy as np @@ -28,7 +28,13 @@ from pandas.core import algorithms from pandas.core.accessor import PandasDelegate -from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin +from pandas.core.arrays import ( + DatetimeArray, + ExtensionArray, + ExtensionOpsMixin, + PeriodArray, + TimedeltaArray, +) from pandas.core.arrays.datetimelike import ( DatetimeLikeArrayMixin, _ensure_datetimelike_to_i8, @@ -91,7 +97,7 @@ class DatetimeIndexOpsMixin(ExtensionIndex, ExtensionOpsMixin): Common ops mixin to support a unified interface datetimelike Index. """ - _data: ExtensionArray + _data: Union[DatetimeArray, TimedeltaArray, PeriodArray] freq: Optional[DateOffset] freqstr: Optional[str] _resolution: int From b8895e7769e248636034e67dcf587373c7b3acf4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 6 Jan 2020 15:11:15 -0800 Subject: [PATCH 3/7] fix coercion test --- pandas/tests/indexing/test_coercion.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index 8843f6c08fe80..0ebe1678d9f3e 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -432,13 +432,19 @@ def test_insert_index_datetimes(self, fill_val, exp_dtype): ) self._assert_insert_conversion(obj, fill_val, exp, exp_dtype) - msg = "Passed item and index have different timezone" if fill_val.tz: - with pytest.raises(ValueError, match=msg): + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): obj.insert(1, pd.Timestamp("2012-01-01")) - with pytest.raises(ValueError, match=msg): - obj.insert(1, pd.Timestamp("2012-01-01", tz="Asia/Tokyo")) + msg = "Timezones don't match" + with pytest.raises(ValueError, match=msg): + obj.insert(1, pd.Timestamp("2012-01-01", tz="Asia/Tokyo")) + + else: + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + obj.insert(1, pd.Timestamp("2012-01-01", tz="Asia/Tokyo")) msg = "cannot insert DatetimeIndex with incompatible label" with pytest.raises(TypeError, match=msg): From 40f00b96e3ec4e0935425ecb6cf5c945237514ed Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 6 Jan 2020 17:58:22 -0800 Subject: [PATCH 4/7] troubleshoot mypy --- pandas/core/indexes/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 7d792c4fbbc3c..cae84650d5219 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -951,7 +951,7 @@ class DatetimelikeDelegateMixin(PandasDelegate): _raw_methods: Set[str] = set() # raw_properties : dispatch properties that shouldn't be boxed in an Index _raw_properties: Set[str] = set() - _data: ExtensionArray + _data: Union[DatetimeArray, TimedeltaArray, PeriodArray] def _delegate_property_get(self, name, *args, **kwargs): result = getattr(self._data, name) From 29f26f3c2a869b398491da182e92f8a778dbdbc6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 6 Jan 2020 18:29:44 -0800 Subject: [PATCH 5/7] remove unused import --- pandas/core/indexes/datetimelike.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index cae84650d5219..717a4fa12921a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -30,7 +30,6 @@ from pandas.core.accessor import PandasDelegate from pandas.core.arrays import ( DatetimeArray, - ExtensionArray, ExtensionOpsMixin, PeriodArray, TimedeltaArray, From c203c2c49faeffaf5c97c09f8c5d56a6b05547d2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 8 Jan 2020 10:41:37 -0800 Subject: [PATCH 6/7] troubleshoot mypy --- pandas/core/indexes/datetimes.py | 2 +- pandas/core/indexes/period.py | 2 +- pandas/core/indexes/timedeltas.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index d186dc5af9faf..ba62e13c35579 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -872,7 +872,7 @@ def inferred_type(self) -> str: # sure we can't have ambiguous indexing return "datetime64" - def insert(self, loc: int, item): + def insert(self, loc, item): """ Make new Index inserting new item at location diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index d97f0a0535843..d34ac1a541d27 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -714,7 +714,7 @@ def _convert_tolerance(self, tolerance, target): raise ValueError("list-like tolerance size must match target index size") return self._maybe_convert_timedelta(tolerance) - def insert(self, loc: int, item): + def insert(self, loc, item): if not isinstance(item, Period) or self.freq != item.freq: return self.astype(object).insert(loc, item) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 1e1404799bf7a..13fd14dafa27b 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -370,7 +370,7 @@ def is_type_compatible(self, typ) -> bool: def inferred_type(self) -> str: return "timedelta64" - def insert(self, loc: int, item): + def insert(self, loc, item): """ Make new Index inserting new item at location From 5a6def3e15693dc78b65c37f9195486a0d7d94f8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 8 Jan 2020 11:04:52 -0800 Subject: [PATCH 7/7] troubleshoot mypy --- pandas/core/indexes/datetimelike.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 5f89dd48e69aa..53eef9d195af4 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -2,7 +2,7 @@ Base and utility classes for tseries type pandas objects. """ import operator -from typing import List, Optional, Set, Union +from typing import List, Optional, Set import numpy as np @@ -33,8 +33,8 @@ from pandas.core.accessor import PandasDelegate from pandas.core.arrays import ( DatetimeArray, + ExtensionArray, ExtensionOpsMixin, - PeriodArray, TimedeltaArray, ) from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin @@ -96,7 +96,7 @@ class DatetimeIndexOpsMixin(ExtensionIndex, ExtensionOpsMixin): Common ops mixin to support a unified interface datetimelike Index. """ - _data: Union[DatetimeArray, TimedeltaArray, PeriodArray] + _data: ExtensionArray freq: Optional[DateOffset] freqstr: Optional[str] _resolution: int @@ -925,7 +925,7 @@ class DatetimelikeDelegateMixin(PandasDelegate): _raw_methods: Set[str] = set() # raw_properties : dispatch properties that shouldn't be boxed in an Index _raw_properties: Set[str] = set() - _data: Union[DatetimeArray, TimedeltaArray, PeriodArray] + _data: ExtensionArray def _delegate_property_get(self, name, *args, **kwargs): result = getattr(self._data, name)