From 6d236258f9690ec94a50cb69f369c2f572bb4b05 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Thu, 8 Jun 2017 09:40:32 -0400 Subject: [PATCH 01/12] BUG: TimedeltaIndex raising ValueError when slice indexing (#16637) closes #16637 --- doc/source/whatsnew/v0.20.2.txt | 1 + pandas/core/indexes/timedeltas.py | 3 +-- pandas/tests/indexing/test_timedelta.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.20.2.txt b/doc/source/whatsnew/v0.20.2.txt index 31125db0f34d4..f1093e712aeb9 100644 --- a/doc/source/whatsnew/v0.20.2.txt +++ b/doc/source/whatsnew/v0.20.2.txt @@ -51,6 +51,7 @@ Bug Fixes - Bug in :func:`unique` on an array of tuples (:issue:`16519`) - Bug in :func:`cut` when ``labels`` are set, resulting in incorrect label ordering (:issue:`16459`) - Fixed a compatibility issue with IPython 6.0's tab completion showing deprecation warnings on ``Categoricals`` (:issue:`16409`) +- Bug in ``TimedeltaIndex`` raising a ``ValueError`` when slice indexing with ``loc`` (:issue:`16637`) Conversion ^^^^^^^^^^ diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index ab94a5bffb4f9..c025c74625972 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -680,8 +680,7 @@ def get_loc(self, key, method=None, tolerance=None): ------- loc : int """ - - if is_bool_indexer(key): + if is_bool_indexer(key) or is_timedelta64_dtype(key): raise TypeError if isnull(key): diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index cf8cc6c2d345d..5cba019e6e02d 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -18,3 +18,20 @@ def test_boolean_indexing(self): index=pd.to_timedelta(range(10), unit='s'), columns=['x']) tm.assert_frame_equal(expected, result) + + def test_slice_indexing(self): + # GH 16637 + df = pd.DataFrame({'x': range(10)}) + df.index = pd.to_timedelta(range(10), unit='s') + + conditions = [df.index[0], df.index[4:8], df.index[[3, 5]]] + expected_data = [[20, 1, 2, 3, 4, 5, 6, 7, 8, 9], + [0, 1, 2, 3, 20, 20, 20, 20, 8, 9], + [0, 1, 2, 20, 4, 20, 6, 7, 8, 9]] + for cond, data in zip(conditions, expected_data): + result = df.copy() + result.loc[cond, 'x'] = 20 + expected = pd.DataFrame(data, + index=pd.to_timedelta(range(10), unit='s'), + columns=['x']) + tm.assert_frame_equal(expected, result) From 630987724ab3722d4298589fac8cd0615c72b726 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Thu, 8 Jun 2017 09:51:47 -0400 Subject: [PATCH 02/12] v0.20.2 is already released --- doc/source/whatsnew/v0.20.2.txt | 1 - doc/source/whatsnew/v0.20.3.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.20.2.txt b/doc/source/whatsnew/v0.20.2.txt index f1093e712aeb9..31125db0f34d4 100644 --- a/doc/source/whatsnew/v0.20.2.txt +++ b/doc/source/whatsnew/v0.20.2.txt @@ -51,7 +51,6 @@ Bug Fixes - Bug in :func:`unique` on an array of tuples (:issue:`16519`) - Bug in :func:`cut` when ``labels`` are set, resulting in incorrect label ordering (:issue:`16459`) - Fixed a compatibility issue with IPython 6.0's tab completion showing deprecation warnings on ``Categoricals`` (:issue:`16409`) -- Bug in ``TimedeltaIndex`` raising a ``ValueError`` when slice indexing with ``loc`` (:issue:`16637`) Conversion ^^^^^^^^^^ diff --git a/doc/source/whatsnew/v0.20.3.txt b/doc/source/whatsnew/v0.20.3.txt index dcce427f8dd84..f11d4d866de5c 100644 --- a/doc/source/whatsnew/v0.20.3.txt +++ b/doc/source/whatsnew/v0.20.3.txt @@ -41,6 +41,7 @@ Bug Fixes - Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`) - Fixed compat with loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`) - Fixed bug where computing the rolling covariance of a MultiIndexed ``DataFrame`` improperly raised a ``ValueError`` (:issue:`16789`) +- Bug in ``TimedeltaIndex`` raising a ``ValueError`` when slice indexing with ``loc`` (:issue:`16637`) Conversion ^^^^^^^^^^ From 094d25ad831caa448e189d645e2edcc973fa7ad2 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Thu, 8 Jun 2017 12:54:25 -0400 Subject: [PATCH 03/12] Fixed unit tests. --- pandas/core/dtypes/common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index bfec1ec3ebe8c..2eebf3704253e 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -396,7 +396,10 @@ def is_timedelta64_dtype(arr_or_dtype): if arr_or_dtype is None: return False - tipo = _get_dtype_type(arr_or_dtype) + try: + tipo = _get_dtype_type(arr_or_dtype) + except ValueError: + return False return issubclass(tipo, np.timedelta64) From 4a0b379821988a6fe4e9c45b2fd96de7e0db38cf Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Thu, 8 Jun 2017 13:15:04 -0400 Subject: [PATCH 04/12] This test would fail on 32 bits. --- pandas/tests/indexing/test_timedelta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index 5cba019e6e02d..80c9889a0fa52 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -13,7 +13,7 @@ def test_boolean_indexing(self): [0, 1, 2, 10, 4, 5, 6, 7, 8, 9], [10, 10, 10, 3, 4, 5, 6, 7, 8, 9]] for cond, data in zip(conditions, expected_data): - result = df.assign(x=df.mask(cond, 10).astype('int64')) + result = df.assign(x=df.mask(cond, 10).astype(df['x'].dtype)) expected = pd.DataFrame(data, index=pd.to_timedelta(range(10), unit='s'), columns=['x']) From d842230f13b086777117305eeb5f5da37b3a667c Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Fri, 9 Jun 2017 11:28:03 -0400 Subject: [PATCH 05/12] Changes based on review comments * Fixed bug on windows with int32/int64 confusion * Added unit_test to is_timedelta64_dtype when a string is passed * Changed unit_test name from `test_slice_indexing` to `test_list_like_indexing` --- pandas/tests/dtypes/test_common.py | 2 ++ pandas/tests/indexing/test_timedelta.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 4633dde5ed537..326ff7119949c 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -204,6 +204,8 @@ def test_is_timedelta64_dtype(): assert com.is_timedelta64_dtype(np.timedelta64) assert com.is_timedelta64_dtype(pd.Series([], dtype="timedelta64[ns]")) + assert not com.is_timedelta64_dtype("0 days 00:00:00") + def test_is_period_dtype(): assert not com.is_period_dtype(object) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index 80c9889a0fa52..050c9c45ca66a 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -13,13 +13,14 @@ def test_boolean_indexing(self): [0, 1, 2, 10, 4, 5, 6, 7, 8, 9], [10, 10, 10, 3, 4, 5, 6, 7, 8, 9]] for cond, data in zip(conditions, expected_data): - result = df.assign(x=df.mask(cond, 10).astype(df['x'].dtype)) + result = df.assign(x=df.mask(cond, 10).astype('int64')) expected = pd.DataFrame(data, index=pd.to_timedelta(range(10), unit='s'), - columns=['x']) + columns=['x'], + dtype='int64') tm.assert_frame_equal(expected, result) - def test_slice_indexing(self): + def test_list_like_indexing(self): # GH 16637 df = pd.DataFrame({'x': range(10)}) df.index = pd.to_timedelta(range(10), unit='s') From 00675c067abfd5e5d556b4113589db0c055cc601 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Fri, 9 Jun 2017 12:53:05 -0400 Subject: [PATCH 06/12] * Fixed bug on windows with int32/int64 confusion --- pandas/tests/indexing/test_timedelta.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index 050c9c45ca66a..f97951f85f720 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -22,7 +22,7 @@ def test_boolean_indexing(self): def test_list_like_indexing(self): # GH 16637 - df = pd.DataFrame({'x': range(10)}) + df = pd.DataFrame({'x': range(10)}, dtype="int64") df.index = pd.to_timedelta(range(10), unit='s') conditions = [df.index[0], df.index[4:8], df.index[[3, 5]]] @@ -34,5 +34,6 @@ def test_list_like_indexing(self): result.loc[cond, 'x'] = 20 expected = pd.DataFrame(data, index=pd.to_timedelta(range(10), unit='s'), - columns=['x']) + columns=['x'], + dtype="int64") tm.assert_frame_equal(expected, result) From 12ecdc423241164281f32f632f1f23828fb28831 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Fri, 9 Jun 2017 19:11:54 -0400 Subject: [PATCH 07/12] Final change based on @jreback comments --- doc/source/whatsnew/v0.20.3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.20.3.txt b/doc/source/whatsnew/v0.20.3.txt index f11d4d866de5c..92e250c7bdef8 100644 --- a/doc/source/whatsnew/v0.20.3.txt +++ b/doc/source/whatsnew/v0.20.3.txt @@ -41,7 +41,7 @@ Bug Fixes - Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`) - Fixed compat with loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`) - Fixed bug where computing the rolling covariance of a MultiIndexed ``DataFrame`` improperly raised a ``ValueError`` (:issue:`16789`) -- Bug in ``TimedeltaIndex`` raising a ``ValueError`` when slice indexing with ``loc`` (:issue:`16637`) +- Bug in a DataFrame/Series with a ``TimedeltaIndex`` when slice indexing (:issue:`16637`) Conversion ^^^^^^^^^^ From dc8cb7a0d26983e13a7f7aad76c48eefea81be83 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Fri, 9 Jun 2017 19:23:58 -0400 Subject: [PATCH 08/12] Using pytest parametrize as per @jreback suggestion. --- pandas/tests/indexing/test_timedelta.py | 30 ++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index f97951f85f720..43285280e5a63 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -1,3 +1,5 @@ +import pytest + import pandas as pd from pandas.util import testing as tm @@ -20,20 +22,22 @@ def test_boolean_indexing(self): dtype='int64') tm.assert_frame_equal(expected, result) - def test_list_like_indexing(self): + @pytest.mark.parametrize("indexer, expected", + [(0, [20, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + (slice(4, 8), [0, 1, 2, 3, 20, 20, 20, 20, 8, 9],), + ([3, 5], [0, 1, 2, 20, 4, 20, 6, 7, 8, 9])]) + def test_list_like_indexing(self, indexer, expected): # GH 16637 df = pd.DataFrame({'x': range(10)}, dtype="int64") df.index = pd.to_timedelta(range(10), unit='s') + try: + df.loc[df.index[indexer], 'x'] = 20 + except ValueError: + raise ValueError(indexer) - conditions = [df.index[0], df.index[4:8], df.index[[3, 5]]] - expected_data = [[20, 1, 2, 3, 4, 5, 6, 7, 8, 9], - [0, 1, 2, 3, 20, 20, 20, 20, 8, 9], - [0, 1, 2, 20, 4, 20, 6, 7, 8, 9]] - for cond, data in zip(conditions, expected_data): - result = df.copy() - result.loc[cond, 'x'] = 20 - expected = pd.DataFrame(data, - index=pd.to_timedelta(range(10), unit='s'), - columns=['x'], - dtype="int64") - tm.assert_frame_equal(expected, result) + expected = pd.DataFrame(expected, + index=pd.to_timedelta(range(10), unit='s'), + columns=['x'], + dtype="int64") + + tm.assert_frame_equal(expected, df) From 358bfcaaf1161d3d055b86a23fdd6a0bb759cb7b Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Mon, 12 Jun 2017 11:39:00 -0400 Subject: [PATCH 09/12] Unneeded try except. --- pandas/tests/indexing/test_timedelta.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index 43285280e5a63..6be9d46ed99d7 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -30,10 +30,8 @@ def test_list_like_indexing(self, indexer, expected): # GH 16637 df = pd.DataFrame({'x': range(10)}, dtype="int64") df.index = pd.to_timedelta(range(10), unit='s') - try: - df.loc[df.index[indexer], 'x'] = 20 - except ValueError: - raise ValueError(indexer) + + df.loc[df.index[indexer], 'x'] = 20 expected = pd.DataFrame(expected, index=pd.to_timedelta(range(10), unit='s'), From 186549aef3ec67f88fcf490466a44b505137c3b3 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Mon, 3 Jul 2017 11:03:44 -0400 Subject: [PATCH 10/12] Corrections from @jreback comments * Added test for is_timedelta64_dtype * Added double backticks in what's new --- pandas/tests/dtypes/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 326ff7119949c..ba510e68f9a21 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -200,7 +200,7 @@ def test_is_datetime64tz_dtype(): def test_is_timedelta64_dtype(): assert not com.is_timedelta64_dtype(object) assert not com.is_timedelta64_dtype([1, 2, 3]) - + assert not com.is_timedelta64_dtype(np.array([], dtype=np.datetime64)) assert com.is_timedelta64_dtype(np.timedelta64) assert com.is_timedelta64_dtype(pd.Series([], dtype="timedelta64[ns]")) From 7df27da8e6dd4aaffc1bf985d191ffae1365ece3 Mon Sep 17 00:00:00 2001 From: Jean-Mathieu Deschenes Date: Mon, 3 Jul 2017 14:36:33 -0400 Subject: [PATCH 11/12] Fixed lint issue --- pandas/tests/indexing/test_timedelta.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py index 6be9d46ed99d7..be3ea8f0c371d 100644 --- a/pandas/tests/indexing/test_timedelta.py +++ b/pandas/tests/indexing/test_timedelta.py @@ -22,10 +22,11 @@ def test_boolean_indexing(self): dtype='int64') tm.assert_frame_equal(expected, result) - @pytest.mark.parametrize("indexer, expected", - [(0, [20, 1, 2, 3, 4, 5, 6, 7, 8, 9]), - (slice(4, 8), [0, 1, 2, 3, 20, 20, 20, 20, 8, 9],), - ([3, 5], [0, 1, 2, 20, 4, 20, 6, 7, 8, 9])]) + @pytest.mark.parametrize( + "indexer, expected", + [(0, [20, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + (slice(4, 8), [0, 1, 2, 3, 20, 20, 20, 20, 8, 9]), + ([3, 5], [0, 1, 2, 20, 4, 20, 6, 7, 8, 9])]) def test_list_like_indexing(self, indexer, expected): # GH 16637 df = pd.DataFrame({'x': range(10)}, dtype="int64") From b5f43a08898733e6cc58fe29a578267d81104534 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 5 Jul 2017 09:53:17 -0500 Subject: [PATCH 12/12] Update v0.20.3.txt Fixed duplicate line in readme --- doc/source/whatsnew/v0.20.3.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v0.20.3.txt b/doc/source/whatsnew/v0.20.3.txt index 568a4aa398bea..ce7e8be16d8e2 100644 --- a/doc/source/whatsnew/v0.20.3.txt +++ b/doc/source/whatsnew/v0.20.3.txt @@ -40,7 +40,6 @@ Bug Fixes - Fixed issue with :meth:`DataFrame.style` where element id's were not unique (:issue:`16780`) - Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`) - Fixed compat with loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`) -- Fixed bug where computing the rolling covariance of a MultiIndexed ``DataFrame`` improperly raised a ``ValueError`` (:issue:`16789`) - Fixed a bug in failing to compute rolling computations of a column-MultiIndexed ``DataFrame`` (:issue:`16789`, :issue:`16825`) - Bug in a DataFrame/Series with a ``TimedeltaIndex`` when slice indexing (:issue:`16637`)