From 24a0149dd23618cb6a661ddd8486475bc010cbf7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 10 Jul 2019 07:49:46 -0700 Subject: [PATCH 1/6] Fix+test wrong-dtype nat assigned to Series --- pandas/_libs/index.pyx | 4 +++ pandas/core/internals/blocks.py | 3 +- pandas/core/series.py | 3 +- pandas/tests/series/indexing/test_indexing.py | 31 +++++++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx index ba2838d59f814..7000c07b1f5a6 100644 --- a/pandas/_libs/index.pyx +++ b/pandas/_libs/index.pyx @@ -544,6 +544,10 @@ cpdef convert_scalar(ndarray arr, object value): pass elif isinstance(value, timedelta): return Timedelta(value).value + elif util.is_datetime64_object(value): + # exclude np.datetime64("NaT") which would otherwise be picked up + # by the `value != value check below + pass elif value is None or value != value: return NPY_NAT elif isinstance(value, str): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 36074e19240bd..1507aa27d7daa 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2665,7 +2665,8 @@ def _try_coerce_args(self, other): base-type other """ - if is_null_datetimelike(other): + if is_null_datetimelike(other) and not isinstance(other, np.datetime64): + # exclude np.datetime64("NaT") other = tslibs.iNaT elif isinstance(other, (timedelta, np.timedelta64)): other = Timedelta(other).value diff --git a/pandas/core/series.py b/pandas/core/series.py index b3a7f38aef8ef..356dfe4b6dcb4 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1210,7 +1210,8 @@ def setitem(key, value): pass elif is_timedelta64_dtype(self.dtype): # reassign a null value to iNaT - if isna(value): + if isna(value) and not isinstance(value, np.datetime64): + # exclude np.datetime64("NaT") value = iNaT try: diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c8342c54e9b5d..044463621cb5f 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -654,6 +654,37 @@ def test_timedelta_assignment(): tm.assert_series_equal(s, expected) +class TestNaTAssignment: + @pytest.mark.parametrize( + "nat_val,should_cast", + [ + (pd.NaT, True), + (np.timedelta64("NaT", "ns"), True), + (np.datetime64("NaT", "ns"), False), + ], + ) + def test_series_assign_nat_valid(self, nat_val, should_cast): + # some nat-like values should be cast to timedelta64 when inserting + # into a timedelta64 series. Others should coerce to object + # and retain their dtypes. + base = pd.Series([0, 1, 2], dtype="m8[ns]") + expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]") + if not should_cast: + expected = expected.astype(object) + + ser = base.copy(deep=True) + ser[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat_val + tm.assert_series_equal(ser, expected) + + def test_underlying_data_conversion(): # GH 4080 df = DataFrame({c: [1, 2, 3] for c in ["a", "b", "c"]}) From 0e4de0328bb3e1a33ad01afc18584027c3e22585 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 10 Jul 2019 07:50:13 -0700 Subject: [PATCH 2/6] rename test --- pandas/tests/series/indexing/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 044463621cb5f..5b5178386d3b4 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -663,7 +663,7 @@ class TestNaTAssignment: (np.datetime64("NaT", "ns"), False), ], ) - def test_series_assign_nat_valid(self, nat_val, should_cast): + def test_td64_series_assign_nat(self, nat_val, should_cast): # some nat-like values should be cast to timedelta64 when inserting # into a timedelta64 series. Others should coerce to object # and retain their dtypes. From 80a558d1c8ead9e3db444c9c82a290d4af16c90a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 10 Jul 2019 20:30:39 -0700 Subject: [PATCH 3/6] catch ValueError bc it comes up in some builds --- pandas/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index 03c69654f0e69..64429f60ee680 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1218,7 +1218,7 @@ def setitem(key, value): try: self.index._engine.set_value(self._values, key, value) return - except TypeError: + except (TypeError, ValueError): pass self.loc[key] = value From 8de3b74eb77807461f8f8c99660b64c0e8061948 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 12 Jul 2019 18:14:24 -0700 Subject: [PATCH 4/6] change test class to func --- pandas/tests/series/indexing/test_indexing.py | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 2d05f99503886..d73be76795c88 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -654,35 +654,34 @@ def test_timedelta_assignment(): tm.assert_series_equal(s, expected) -class TestNaTAssignment: - @pytest.mark.parametrize( - "nat_val,should_cast", - [ - (pd.NaT, True), - (np.timedelta64("NaT", "ns"), True), - (np.datetime64("NaT", "ns"), False), - ], - ) - def test_td64_series_assign_nat(self, nat_val, should_cast): - # some nat-like values should be cast to timedelta64 when inserting - # into a timedelta64 series. Others should coerce to object - # and retain their dtypes. - base = pd.Series([0, 1, 2], dtype="m8[ns]") - expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]") - if not should_cast: - expected = expected.astype(object) - - ser = base.copy(deep=True) - ser[0] = nat_val - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.loc[0] = nat_val - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.iloc[0] = nat_val - tm.assert_series_equal(ser, expected) +@pytest.mark.parametrize( + "nat_val,should_cast", + [ + (pd.NaT, True), + (np.timedelta64("NaT", "ns"), True), + (np.datetime64("NaT", "ns"), False), + ], +) +def test_td64_series_assign_nat(nat_val, should_cast): + # some nat-like values should be cast to timedelta64 when inserting + # into a timedelta64 series. Others should coerce to object + # and retain their dtypes. + base = pd.Series([0, 1, 2], dtype="m8[ns]") + expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]") + if not should_cast: + expected = expected.astype(object) + + ser = base.copy(deep=True) + ser[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat_val + tm.assert_series_equal(ser, expected) @pytest.mark.parametrize( From f101db70238656a1db5b5c5dd21cdf010de615a0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 15 Jul 2019 14:50:46 -0700 Subject: [PATCH 5/6] use is_valid_nat_for_dtype --- pandas/core/series.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index ce9478e0697e8..1843f0216642d 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -48,6 +48,7 @@ from pandas.core.dtypes.missing import ( isna, na_value_for_dtype, + is_valid_nat_for_dtype, notna, remove_na_arraylike, ) @@ -1198,7 +1199,7 @@ def setitem(key, value): pass elif is_timedelta64_dtype(self.dtype): # reassign a null value to iNaT - if isna(value) and not isinstance(value, np.datetime64): + if is_valid_nat_for_dtype(value, self.dtype): # exclude np.datetime64("NaT") value = iNaT @@ -1206,6 +1207,7 @@ def setitem(key, value): self.index._engine.set_value(self._values, key, value) return except (TypeError, ValueError): + # ValueError appears in only some builds in CI pass self.loc[key] = value From bebcedb5cbe5d3943f51bc9ee37b0ca29ba10b79 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 15 Jul 2019 15:47:27 -0700 Subject: [PATCH 6/6] isort fixup --- pandas/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index 1843f0216642d..dc9a42b7071cb 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -46,9 +46,9 @@ ABCSparseSeries, ) from pandas.core.dtypes.missing import ( + is_valid_nat_for_dtype, isna, na_value_for_dtype, - is_valid_nat_for_dtype, notna, remove_na_arraylike, )