From 68049261f6d7b7c34fe83a0f0654f614d1832f8f Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Wed, 1 Jan 2020 21:31:46 +0200 Subject: [PATCH 1/7] Added 'pearson' to methods list --- pandas/core/nanops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 584972f2b2dd5..591ad88c0d5a3 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -1241,7 +1241,7 @@ def nancorr(a, b, method="pearson", min_periods=None): def get_corr_func(method): - if method in ["kendall", "spearman"]: + if method in ["kendall", "spearman", "pearson"]: from scipy.stats import kendalltau, spearmanr elif callable(method): return method From 0f1a2f22c4ff1079de99e83a021d623d310e043e Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Wed, 1 Jan 2020 22:36:12 +0200 Subject: [PATCH 2/7] Added 'scipy' as a dependency for azure-macos --- ci/deps/azure-macos-36.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/deps/azure-macos-36.yaml b/ci/deps/azure-macos-36.yaml index f393ed84ecf63..ea0083d1c8308 100644 --- a/ci/deps/azure-macos-36.yaml +++ b/ci/deps/azure-macos-36.yaml @@ -26,6 +26,7 @@ dependencies: - pytables - python-dateutil==2.6.1 - pytz + - scipy>=1.1 - xarray - xlrd - xlsxwriter From a7decf4378945b128098031ca3193275ba4710b9 Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Wed, 1 Jan 2020 23:39:08 +0200 Subject: [PATCH 3/7] Removed 'scipy' dependency for azure-macos --- ci/deps/azure-macos-36.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/deps/azure-macos-36.yaml b/ci/deps/azure-macos-36.yaml index ea0083d1c8308..f393ed84ecf63 100644 --- a/ci/deps/azure-macos-36.yaml +++ b/ci/deps/azure-macos-36.yaml @@ -26,7 +26,6 @@ dependencies: - pytables - python-dateutil==2.6.1 - pytz - - scipy>=1.1 - xarray - xlrd - xlsxwriter From ed6f9c951be18edfdd90a659ae3f833ba8dcbbe0 Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Wed, 1 Jan 2020 23:43:12 +0200 Subject: [PATCH 4/7] Added skip if no scipy decorator --- pandas/tests/frame/methods/test_cov_corr.py | 3 +++ pandas/tests/series/test_timeseries.py | 1 + pandas/tests/test_nanops.py | 2 ++ pandas/tests/window/moments/test_moments_rolling.py | 1 + 4 files changed, 7 insertions(+) diff --git a/pandas/tests/frame/methods/test_cov_corr.py b/pandas/tests/frame/methods/test_cov_corr.py index 209b4a800354d..c6b48e95cc4ad 100644 --- a/pandas/tests/frame/methods/test_cov_corr.py +++ b/pandas/tests/frame/methods/test_cov_corr.py @@ -155,6 +155,7 @@ def test_corr_int(self): class TestDataFrameCorrWith: + @td.skip_if_no_scipy def test_corrwith(self, datetime_frame): a = datetime_frame noise = Series(np.random.randn(len(a)), index=a.index) @@ -203,6 +204,7 @@ def test_corrwith_with_objects(self): expected = df1.loc[:, cols].corrwith(df2.loc[:, cols], axis=1) tm.assert_series_equal(result, expected) + @td.skip_if_no_scipy def test_corrwith_series(self, datetime_frame): result = datetime_frame.corrwith(datetime_frame["A"]) expected = datetime_frame.apply(datetime_frame["A"].corr) @@ -218,6 +220,7 @@ def test_corrwith_matches_corrcoef(self): tm.assert_almost_equal(c1, c2) assert c1 < 1 + @td.skip_if_no_scipy def test_corrwith_mixed_dtypes(self): # GH#18570 df = pd.DataFrame( diff --git a/pandas/tests/series/test_timeseries.py b/pandas/tests/series/test_timeseries.py index c3e5e8b975cda..85efead123695 100644 --- a/pandas/tests/series/test_timeseries.py +++ b/pandas/tests/series/test_timeseries.py @@ -76,6 +76,7 @@ def test_asfreq_datetimeindex_empty_series(self): result = Series([3], index=index.copy()).asfreq("H") tm.assert_index_equal(expected.index, result.index) + @td.skip_if_no_scipy def test_autocorr(self, datetime_series): # Just run the function corr1 = datetime_series.autocorr() diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index c207c803510ca..2117107e9c50a 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -560,6 +560,7 @@ def check_nancorr_nancov_1d(self, checkfun, targ0, targ1, **kwargs): tm.assert_almost_equal(targ2, res24) tm.assert_almost_equal(targ2, res25) + @td.skip_if_no_scipy def test_nancorr(self): targ0 = np.corrcoef(self.arr_float_2d, self.arr_float1_2d)[0, 1] targ1 = np.corrcoef(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1] @@ -568,6 +569,7 @@ def test_nancorr(self): targ1 = np.corrcoef(self.arr_float_1d.flat, self.arr_float1_1d.flat)[0, 1] self.check_nancorr_nancov_1d(nanops.nancorr, targ0, targ1, method="pearson") + @td.skip_if_no_scipy def test_nancorr_pearson(self): targ0 = np.corrcoef(self.arr_float_2d, self.arr_float1_2d)[0, 1] targ1 = np.corrcoef(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1] diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index c110ed172ecb9..6d68e267dcc4a 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1097,6 +1097,7 @@ def test_rolling_cov(self): def test_rolling_cov_pairwise(self): self._check_pairwise_moment("rolling", "cov", window=10, min_periods=5) + @td.skip_if_no_scipy def test_rolling_corr(self): A = self.series B = A + randn(len(A)) From 51d59016e58be3488799af9ba07f3d4580b09175 Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Thu, 2 Jan 2020 03:38:08 +0200 Subject: [PATCH 5/7] Fixes for review #1 --- pandas/core/nanops.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 591ad88c0d5a3..6b03e76a1d691 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -1241,10 +1241,16 @@ def nancorr(a, b, method="pearson", min_periods=None): def get_corr_func(method): - if method in ["kendall", "spearman", "pearson"]: + if method in ["kendall", "spearman"]: from scipy.stats import kendalltau, spearmanr + elif method in ["pearson"]: + pass elif callable(method): return method + else: + raise ValueError( + f"Unkown method '{method}', expected one of 'kendall', 'spearman'" + ) def _pearson(a, b): return np.corrcoef(a, b)[0, 1] From 866c70e72309fa3b9a704f2a6b560f3a02f56721 Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Thu, 2 Jan 2020 03:42:16 +0200 Subject: [PATCH 6/7] Removed all added skips --- pandas/tests/frame/methods/test_cov_corr.py | 3 --- pandas/tests/series/test_timeseries.py | 1 - pandas/tests/test_nanops.py | 2 -- pandas/tests/window/moments/test_moments_rolling.py | 1 - 4 files changed, 7 deletions(-) diff --git a/pandas/tests/frame/methods/test_cov_corr.py b/pandas/tests/frame/methods/test_cov_corr.py index c6b48e95cc4ad..209b4a800354d 100644 --- a/pandas/tests/frame/methods/test_cov_corr.py +++ b/pandas/tests/frame/methods/test_cov_corr.py @@ -155,7 +155,6 @@ def test_corr_int(self): class TestDataFrameCorrWith: - @td.skip_if_no_scipy def test_corrwith(self, datetime_frame): a = datetime_frame noise = Series(np.random.randn(len(a)), index=a.index) @@ -204,7 +203,6 @@ def test_corrwith_with_objects(self): expected = df1.loc[:, cols].corrwith(df2.loc[:, cols], axis=1) tm.assert_series_equal(result, expected) - @td.skip_if_no_scipy def test_corrwith_series(self, datetime_frame): result = datetime_frame.corrwith(datetime_frame["A"]) expected = datetime_frame.apply(datetime_frame["A"].corr) @@ -220,7 +218,6 @@ def test_corrwith_matches_corrcoef(self): tm.assert_almost_equal(c1, c2) assert c1 < 1 - @td.skip_if_no_scipy def test_corrwith_mixed_dtypes(self): # GH#18570 df = pd.DataFrame( diff --git a/pandas/tests/series/test_timeseries.py b/pandas/tests/series/test_timeseries.py index 85efead123695..c3e5e8b975cda 100644 --- a/pandas/tests/series/test_timeseries.py +++ b/pandas/tests/series/test_timeseries.py @@ -76,7 +76,6 @@ def test_asfreq_datetimeindex_empty_series(self): result = Series([3], index=index.copy()).asfreq("H") tm.assert_index_equal(expected.index, result.index) - @td.skip_if_no_scipy def test_autocorr(self, datetime_series): # Just run the function corr1 = datetime_series.autocorr() diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 2117107e9c50a..c207c803510ca 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -560,7 +560,6 @@ def check_nancorr_nancov_1d(self, checkfun, targ0, targ1, **kwargs): tm.assert_almost_equal(targ2, res24) tm.assert_almost_equal(targ2, res25) - @td.skip_if_no_scipy def test_nancorr(self): targ0 = np.corrcoef(self.arr_float_2d, self.arr_float1_2d)[0, 1] targ1 = np.corrcoef(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1] @@ -569,7 +568,6 @@ def test_nancorr(self): targ1 = np.corrcoef(self.arr_float_1d.flat, self.arr_float1_1d.flat)[0, 1] self.check_nancorr_nancov_1d(nanops.nancorr, targ0, targ1, method="pearson") - @td.skip_if_no_scipy def test_nancorr_pearson(self): targ0 = np.corrcoef(self.arr_float_2d, self.arr_float1_2d)[0, 1] targ1 = np.corrcoef(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1] diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 6d68e267dcc4a..c110ed172ecb9 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1097,7 +1097,6 @@ def test_rolling_cov(self): def test_rolling_cov_pairwise(self): self._check_pairwise_moment("rolling", "cov", window=10, min_periods=5) - @td.skip_if_no_scipy def test_rolling_corr(self): A = self.series B = A + randn(len(A)) From 260a826fe2cdf27638ca2517e5ff4f6a052e4bc5 Mon Sep 17 00:00:00 2001 From: MomIsBestFriend <> Date: Sun, 5 Jan 2020 09:12:23 +0200 Subject: [PATCH 7/7] TST: Added test case --- pandas/tests/test_nanops.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index b2bccbeb82c27..2c5d028ebe42e 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -598,6 +598,14 @@ def test_nancorr_spearman(self): targ1 = spearmanr(self.arr_float_1d.flat, self.arr_float1_1d.flat)[0] self.check_nancorr_nancov_1d(nanops.nancorr, targ0, targ1, method="spearman") + @td.skip_if_no_scipy + def test_invalid_method(self): + targ0 = np.corrcoef(self.arr_float_2d, self.arr_float1_2d)[0, 1] + targ1 = np.corrcoef(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1] + msg = "Unkown method 'foo', expected one of 'kendall', 'spearman'" + with pytest.raises(ValueError, match=msg): + self.check_nancorr_nancov_1d(nanops.nancorr, targ0, targ1, method="foo") + def test_nancov(self): targ0 = np.cov(self.arr_float_2d, self.arr_float1_2d)[0, 1] targ1 = np.cov(self.arr_float_2d.flat, self.arr_float1_2d.flat)[0, 1]