From 96c45f965681b12acafda1281906fcc2b789af27 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 09:05:43 -0800 Subject: [PATCH 1/8] fix subtraction of non-nano datetime64 from series --- pandas/core/ops.py | 13 +++++++++---- pandas/tests/series/test_operators.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 2fb0cbb14c225..617034fd5b0e5 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -28,7 +28,7 @@ is_datetimelike_v_numeric, is_integer_dtype, is_categorical_dtype, is_object_dtype, is_timedelta64_dtype, - is_datetime64_dtype, is_datetime64tz_dtype, + is_datetime64_dtype, is_datetime64tz_dtype, is_datetime64_ns_dtype, is_bool_dtype, is_datetimetz, is_list_like, is_scalar, @@ -497,6 +497,11 @@ def _convert_to_array(self, values, name=None, other=None): elif not (isinstance(values, (np.ndarray, ABCSeries)) and is_datetime64_dtype(values)): values = libts.array_to_datetime(values) + elif (is_datetime64_dtype(values) and + not is_datetime64_ns_dtype(values)): + # GH#7996 e.g. np.datetime64('2013-01-01') is datetime64[D] + values = values.astype('datetime64[ns]') + elif inferred_type in ('timedelta', 'timedelta64'): # have a timedelta, convert to to ns here values = to_timedelta(values, errors='coerce', box=False) @@ -671,7 +676,7 @@ def _arith_method_SERIES(op, name, str_rep, fill_zeros=None, default_axis=None, """ def na_op(x, y): import pandas.core.computation.expressions as expressions - + # try: result = expressions.evaluate(op, str_rep, x, y, **eval_kwargs) except TypeError: @@ -688,9 +693,9 @@ def na_op(x, y): raise TypeError("{typ} cannot perform the operation " "{op}".format(typ=type(x).__name__, op=str_rep)) - + # result, changed = maybe_upcast_putmask(result, ~mask, np.nan) - + # result = missing.fill_zeros(result, x, y, name, fill_zeros) return result diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 6cc866a35514f..3359871d451e9 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -28,6 +28,25 @@ from .common import TestData +class TestDatetimeLikeArithmetic(object): + def test_sub_datetime64_not_ns(self): + # GH#7996 + ser = Series(date_range('20130101', periods=3)) + dt64 = np.datetime64('2013-01-01') + assert dt64.dtype == 'datetime64[D]' + res = ser - dt64 + expected = pd.Series([Timedelta(days=0), Timedelta(days=1), + Timedelta(days=2)]) + tm.assert_series_equal(res, expected) + + # check for DatetimeIndex and DataFrame while we're at it + dti = pd.DatetimeIndex(ser) + res = dti - dt64 + tm.assert_index_equal(res, pd.Index(expected)) + + # TODO: This is still broken for ser.to_frame() + + class TestSeriesOperators(TestData): def test_series_comparison_scalars(self): From 53de7b5e75b8d76c1168119f612af9ab03265bc2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 09:12:02 -0800 Subject: [PATCH 2/8] add test for reverse ops --- pandas/tests/series/test_operators.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 3359871d451e9..7e10c3aa186d8 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -39,11 +39,17 @@ def test_sub_datetime64_not_ns(self): Timedelta(days=2)]) tm.assert_series_equal(res, expected) + res = dt64 - ser + tm.assert_series_equal(res, -expected) + # check for DatetimeIndex and DataFrame while we're at it dti = pd.DatetimeIndex(ser) res = dti - dt64 tm.assert_index_equal(res, pd.Index(expected)) + res = dt64 - dti + tm.assert_index_equal(res, pd.Index(-expected)) + # TODO: This is still broken for ser.to_frame() From 4e32e7fc4ceb8fba4e8c2f9765c00155d10223fa Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 09:12:09 -0800 Subject: [PATCH 3/8] add whatsnew note --- doc/source/whatsnew/v0.22.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index 37032ff6bc313..413d1ff1cda05 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -324,7 +324,7 @@ Reshaping Numeric ^^^^^^^ -- +- Bug in :func:`Series.__sub__` subtracting a non-nanosecond ``np.datetime64`` object from a ``Series`` gave incorrect results (:issue:`7996`) - - From 9480e3a1597c066c0d09b3e9cf211a3b19c60ac8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 15:18:35 -0800 Subject: [PATCH 4/8] remove debugging pounds --- pandas/core/ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 617034fd5b0e5..05f5125112a34 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -676,7 +676,7 @@ def _arith_method_SERIES(op, name, str_rep, fill_zeros=None, default_axis=None, """ def na_op(x, y): import pandas.core.computation.expressions as expressions - # + try: result = expressions.evaluate(op, str_rep, x, y, **eval_kwargs) except TypeError: @@ -693,9 +693,9 @@ def na_op(x, y): raise TypeError("{typ} cannot perform the operation " "{op}".format(typ=type(x).__name__, op=str_rep)) - # + result, changed = maybe_upcast_putmask(result, ~mask, np.nan) - # + result = missing.fill_zeros(result, x, y, name, fill_zeros) return result From aa57961adf4e7218acfba051a5e4f517ea067b2b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 15:19:05 -0800 Subject: [PATCH 5/8] separate xfail test for DataFrame --- pandas/tests/series/test_operators.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 7e10c3aa186d8..bfea202dfd9ea 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -42,7 +42,6 @@ def test_sub_datetime64_not_ns(self): res = dt64 - ser tm.assert_series_equal(res, -expected) - # check for DatetimeIndex and DataFrame while we're at it dti = pd.DatetimeIndex(ser) res = dti - dt64 tm.assert_index_equal(res, pd.Index(expected)) @@ -50,7 +49,15 @@ def test_sub_datetime64_not_ns(self): res = dt64 - dti tm.assert_index_equal(res, pd.Index(-expected)) - # TODO: This is still broken for ser.to_frame() + @pytest.xfail(reason='GH#7996 datetime64 units not converted to nanos') + def test_frame_sub_datetime64_not_ns(self): + df = pd.DataFrame(date_range('20130101', periods=3)) + dt64 = np.datetime64('2013-01-01') + assert dt64.dtype == 'datetime64[D]' + res = df - dt64 + expected = pd.DataFrame([Timedelta(days=0), Timedelta(days=1), + Timedelta(days=2)]) + tm.assert_frame_equal(res, expected) class TestSeriesOperators(TestData): From 61afacc3807b6a321a323e7aa052e5d89955a055 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Dec 2017 15:21:22 -0800 Subject: [PATCH 6/8] mark xfail --- pandas/tests/series/test_operators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index bfea202dfd9ea..82e9bf00a39bc 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -49,7 +49,7 @@ def test_sub_datetime64_not_ns(self): res = dt64 - dti tm.assert_index_equal(res, pd.Index(-expected)) - @pytest.xfail(reason='GH#7996 datetime64 units not converted to nanos') + @pytest.mark.xfail(reason='GH#7996 datetime64 units not converted to nano') def test_frame_sub_datetime64_not_ns(self): df = pd.DataFrame(date_range('20130101', periods=3)) dt64 = np.datetime64('2013-01-01') From b5c16f9af204148e9ca3a3068477f00aebd4b49e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 21 Dec 2017 13:53:08 -0800 Subject: [PATCH 7/8] parametrize per reviewer request --- pandas/tests/series/test_operators.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 37d91ea12ce1e..0fb45e5395ecf 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -962,25 +962,23 @@ def test_timedelta64_ops_nat(self): class TestDatetimeSeriesArithmetic(object): - def test_sub_datetime64_not_ns(self): + @pytest.mark.parametrize('box_cls, assert_func', [(Series, + tm.assert_series_equal), + (pd.Index, + tm.assert_index_equal)]) + def test_sub_datetime64_not_ns(self, box_cls, assert_func): # GH#7996 - ser = Series(date_range('20130101', periods=3)) dt64 = np.datetime64('2013-01-01') assert dt64.dtype == 'datetime64[D]' - res = ser - dt64 - expected = pd.Series([Timedelta(days=0), Timedelta(days=1), - Timedelta(days=2)]) - tm.assert_series_equal(res, expected) - res = dt64 - ser - tm.assert_series_equal(res, -expected) + obj = box_cls(date_range('20130101', periods=3)) + res = obj - dt64 + expected = box_cls([Timedelta(days=0), Timedelta(days=1), + Timedelta(days=2)]) + assert_func(res, expected) - dti = pd.DatetimeIndex(ser) - res = dti - dt64 - tm.assert_index_equal(res, pd.Index(expected)) - - res = dt64 - dti - tm.assert_index_equal(res, pd.Index(-expected)) + res = dt64 - obj + assert_func(res, -expected) @pytest.mark.xfail(reason='GH#7996 datetime64 units not converted to nano') def test_frame_sub_datetime64_not_ns(self): From 0dbe62c6930cf27246f821df5dab0e1de4091f2f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 23 Dec 2017 14:07:48 -0800 Subject: [PATCH 8/8] requested edit --- pandas/tests/series/test_operators.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 28fab9a0e73ce..433e3cf440cbd 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -1004,19 +1004,19 @@ def test_operators_timedelta64_with_timedelta_invalid(self, scalar_td): class TestDatetimeSeriesArithmetic(object): - @pytest.mark.parametrize('box_cls, assert_func', [(Series, - tm.assert_series_equal), - (pd.Index, - tm.assert_index_equal)]) - def test_sub_datetime64_not_ns(self, box_cls, assert_func): + @pytest.mark.parametrize( + 'box, assert_func', + [(Series, tm.assert_series_equal), + (pd.Index, tm.assert_index_equal)]) + def test_sub_datetime64_not_ns(self, box, assert_func): # GH#7996 dt64 = np.datetime64('2013-01-01') assert dt64.dtype == 'datetime64[D]' - obj = box_cls(date_range('20130101', periods=3)) + obj = box(date_range('20130101', periods=3)) res = obj - dt64 - expected = box_cls([Timedelta(days=0), Timedelta(days=1), - Timedelta(days=2)]) + expected = box([Timedelta(days=0), Timedelta(days=1), + Timedelta(days=2)]) assert_func(res, expected) res = dt64 - obj