From 7e461a18d9f6928132afec6f48ce968b3e989ba6 Mon Sep 17 00:00:00 2001 From: Kaiqi Dong Date: Mon, 3 Dec 2018 17:43:52 +0100 Subject: [PATCH 01/13] remove \n from docstring --- pandas/core/arrays/datetimes.py | 26 +++++++++++++------------- pandas/core/arrays/timedeltas.py | 16 ++++++++-------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index cfe3afcf3730a..b3df505d56d78 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -82,7 +82,7 @@ def f(self): return result f.__name__ = name - f.__doc__ = docstring + f.__doc__ = "\n{}\n".format(docstring) return property(f) @@ -1072,19 +1072,19 @@ def date(self): return tslib.ints_to_pydatetime(timestamps, box="date") - year = _field_accessor('year', 'Y', "\n The year of the datetime\n") + year = _field_accessor('year', 'Y', "The year of the datetime") month = _field_accessor('month', 'M', - "\n The month as January=1, December=12 \n") - day = _field_accessor('day', 'D', "\nThe days of the datetime\n") - hour = _field_accessor('hour', 'h', "\nThe hours of the datetime\n") - minute = _field_accessor('minute', 'm', "\nThe minutes of the datetime\n") - second = _field_accessor('second', 's', "\nThe seconds of the datetime\n") + "The month as January=1, December=12") + day = _field_accessor('day', 'D', "The days of the datetime") + hour = _field_accessor('hour', 'h', "The hours of the datetime") + minute = _field_accessor('minute', 'm', "The minutes of the datetime") + second = _field_accessor('second', 's', "The seconds of the datetime") microsecond = _field_accessor('microsecond', 'us', - "\nThe microseconds of the datetime\n") + "The microseconds of the datetime") nanosecond = _field_accessor('nanosecond', 'ns', - "\nThe nanoseconds of the datetime\n") + "The nanoseconds of the datetime") weekofyear = _field_accessor('weekofyear', 'woy', - "\nThe week ordinal of the year\n") + "The week ordinal of the year") week = weekofyear _dayofweek_doc = """ The day of the week with Monday=0, Sunday=6. @@ -1129,12 +1129,12 @@ def date(self): "The name of day in a week (ex: Friday)\n\n.. deprecated:: 0.23.0") dayofyear = _field_accessor('dayofyear', 'doy', - "\nThe ordinal day of the year\n") - quarter = _field_accessor('quarter', 'q', "\nThe quarter of the date\n") + "The ordinal day of the year") + quarter = _field_accessor('quarter', 'q', "The quarter of the date") days_in_month = _field_accessor( 'days_in_month', 'dim', - "\nThe number of days in the month\n") + "The number of days in the month") daysinmonth = days_in_month _is_month_doc = """ Indicates whether the date is the {first_or_last} day of the month. diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 830283d31a929..4afc9f5483c2a 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -59,7 +59,7 @@ def f(self): return result f.__name__ = name - f.__doc__ = docstring + f.__doc__ = "\n{}\n".format(docstring) return property(f) @@ -684,16 +684,16 @@ def to_pytimedelta(self): return tslibs.ints_to_pytimedelta(self.asi8) days = _field_accessor("days", "days", - "\nNumber of days for each element.\n") + "Number of days for each element.") seconds = _field_accessor("seconds", "seconds", - "\nNumber of seconds (>= 0 and less than 1 day) " - "for each element.\n") + "Number of seconds (>= 0 and less than 1 day) " + "for each element.") microseconds = _field_accessor("microseconds", "microseconds", - "\nNumber of microseconds (>= 0 and less " - "than 1 second) for each element.\n") + "Number of microseconds (>= 0 and less " + "than 1 second) for each element.") nanoseconds = _field_accessor("nanoseconds", "nanoseconds", - "\nNumber of nanoseconds (>= 0 and less " - "than 1 microsecond) for each element.\n") + "Number of nanoseconds (>= 0 and less " + "than 1 microsecond) for each element.") @property def components(self): From dea38f24c0067ae3fe9484b837c9649714213bba Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:26:31 +0100 Subject: [PATCH 02/13] fix issue 17038 --- pandas/core/reshape/pivot.py | 4 +++- pandas/tests/reshape/test_pivot.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index b443ba142369c..9743d90f4dd04 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -117,7 +117,9 @@ def pivot_table( agged[v] = maybe_downcast_to_dtype(agged[v], data[v].dtype) table = agged - if table.index.nlevels > 1: + + # GH 17038, this check should only happen if index is specified + if table.index.nlevels > 1 and index: # Related GH #17123 # If index_names are integers, determine whether the integers refer # to the level position or name. diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 743fc50c87e96..46a05123c9fdd 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -896,12 +896,6 @@ def _check_output( totals = table.loc[("All", ""), value_col] assert totals == self.data[value_col].mean() - # no rows - rtable = self.data.pivot_table( - columns=["AA", "BB"], margins=True, aggfunc=np.mean - ) - assert isinstance(rtable, Series) - table = self.data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean") for item in ["DD", "EE", "FF"]: totals = table.loc[("All", ""), item] @@ -972,6 +966,20 @@ def test_pivot_integer_columns(self): tm.assert_frame_equal(table, table2, check_names=False) + @pytest.mark.parametrize("cols", [(1, 2), ("a", "b"), (1, "b"), ("a", 1)]) + def test_pivot_table_multiindex_only(self, cols): + # GH 17038 + df2 = DataFrame({cols[0]: [1, 2, 3], cols[1]: [1, 2, 3], "v": [4, 5, 6]}) + + result = df2.pivot_table(values="v", columns=cols) + expected = DataFrame( + [[4, 5, 6]], + columns=MultiIndex.from_tuples([(1, 1), (2, 2), (3, 3)], names=cols), + index=Index(["v"]), + ) + + tm.assert_frame_equal(result, expected) + def test_pivot_no_level_overlap(self): # GH #1181 From cd9e7ac3f31ffaf95cd628863df911dea9fa1248 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:29:43 +0100 Subject: [PATCH 03/13] revert change --- pandas/core/reshape/pivot.py | 3 +-- pandas/tests/reshape/test_pivot.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 9743d90f4dd04..a7cdbb0da7a4e 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -118,8 +118,7 @@ def pivot_table( table = agged - # GH 17038, this check should only happen if index is specified - if table.index.nlevels > 1 and index: + if table.index.nlevels > 1: # Related GH #17123 # If index_names are integers, determine whether the integers refer # to the level position or name. diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 46a05123c9fdd..743fc50c87e96 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -896,6 +896,12 @@ def _check_output( totals = table.loc[("All", ""), value_col] assert totals == self.data[value_col].mean() + # no rows + rtable = self.data.pivot_table( + columns=["AA", "BB"], margins=True, aggfunc=np.mean + ) + assert isinstance(rtable, Series) + table = self.data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean") for item in ["DD", "EE", "FF"]: totals = table.loc[("All", ""), item] @@ -966,20 +972,6 @@ def test_pivot_integer_columns(self): tm.assert_frame_equal(table, table2, check_names=False) - @pytest.mark.parametrize("cols", [(1, 2), ("a", "b"), (1, "b"), ("a", 1)]) - def test_pivot_table_multiindex_only(self, cols): - # GH 17038 - df2 = DataFrame({cols[0]: [1, 2, 3], cols[1]: [1, 2, 3], "v": [4, 5, 6]}) - - result = df2.pivot_table(values="v", columns=cols) - expected = DataFrame( - [[4, 5, 6]], - columns=MultiIndex.from_tuples([(1, 1), (2, 2), (3, 3)], names=cols), - index=Index(["v"]), - ) - - tm.assert_frame_equal(result, expected) - def test_pivot_no_level_overlap(self): # GH #1181 From e5e912be0f596943067a7df812442764d311a086 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:30:16 +0100 Subject: [PATCH 04/13] revert change --- pandas/core/reshape/pivot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index a7cdbb0da7a4e..b443ba142369c 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -117,7 +117,6 @@ def pivot_table( agged[v] = maybe_downcast_to_dtype(agged[v], data[v].dtype) table = agged - if table.index.nlevels > 1: # Related GH #17123 # If index_names are integers, determine whether the integers refer From d37b2c78c42dd35741b7005809d911084c08a1c9 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sat, 2 May 2020 17:59:13 +0200 Subject: [PATCH 05/13] fixturize functions --- pandas/tests/window/common.py | 34 ------ pandas/tests/window/conftest.py | 47 ++++++++ .../window/moments/test_moments_expanding.py | 88 +++++++-------- .../window/moments/test_moments_rolling.py | 103 +++++++++--------- 4 files changed, 143 insertions(+), 129 deletions(-) diff --git a/pandas/tests/window/common.py b/pandas/tests/window/common.py index 7a7ab57cdb904..25989191ae657 100644 --- a/pandas/tests/window/common.py +++ b/pandas/tests/window/common.py @@ -25,40 +25,6 @@ def _create_data(self): class ConsistencyBase(Base): - base_functions = [ - (lambda v: Series(v).count(), None, "count"), - (lambda v: Series(v).max(), None, "max"), - (lambda v: Series(v).min(), None, "min"), - (lambda v: Series(v).sum(), None, "sum"), - (lambda v: Series(v).mean(), None, "mean"), - (lambda v: Series(v).std(), 1, "std"), - (lambda v: Series(v).cov(Series(v)), None, "cov"), - (lambda v: Series(v).corr(Series(v)), None, "corr"), - (lambda v: Series(v).var(), 1, "var"), - # restore once GH 8086 is fixed - # lambda v: Series(v).skew(), 3, 'skew'), - # (lambda v: Series(v).kurt(), 4, 'kurt'), - # restore once GH 8084 is fixed - # lambda v: Series(v).quantile(0.3), None, 'quantile'), - (lambda v: Series(v).median(), None, "median"), - (np.nanmax, 1, "max"), - (np.nanmin, 1, "min"), - (np.nansum, 1, "sum"), - (np.nanmean, 1, "mean"), - (lambda v: np.nanstd(v, ddof=1), 1, "std"), - (lambda v: np.nanvar(v, ddof=1), 1, "var"), - (np.nanmedian, 1, "median"), - ] - no_nan_functions = [ - (np.max, None, "max"), - (np.min, None, "min"), - (np.sum, None, "sum"), - (np.mean, None, "mean"), - (lambda v: np.std(v, ddof=1), 1, "std"), - (lambda v: np.var(v, ddof=1), 1, "var"), - (np.median, None, "median"), - ] - def _create_data(self): super()._create_data() diff --git a/pandas/tests/window/conftest.py b/pandas/tests/window/conftest.py index 856c8f3884816..074e397675e48 100644 --- a/pandas/tests/window/conftest.py +++ b/pandas/tests/window/conftest.py @@ -242,3 +242,50 @@ def no_nans(x): def consistency_data(request): """Create consistency data""" return request.param + + +@pytest.fixture( + params=[ + (lambda v: Series(v).count(), None, "count"), + (lambda v: Series(v).max(), None, "max"), + (lambda v: Series(v).min(), None, "min"), + (lambda v: Series(v).sum(), None, "sum"), + (lambda v: Series(v).mean(), None, "mean"), + (lambda v: Series(v).std(), 1, "std"), + (lambda v: Series(v).cov(Series(v)), None, "cov"), + (lambda v: Series(v).corr(Series(v)), None, "corr"), + (lambda v: Series(v).var(), 1, "var"), + # restore once GH 8086 is fixed + # lambda v: Series(v).skew(), 3, 'skew'), + # (lambda v: Series(v).kurt(), 4, 'kurt'), + # restore once GH 8084 is fixed + # lambda v: Series(v).quantile(0.3), None, 'quantile'), + (lambda v: Series(v).median(), None, "median"), + (np.nanmax, 1, "max"), + (np.nanmin, 1, "min"), + (np.nansum, 1, "sum"), + (np.nanmean, 1, "mean"), + (lambda v: np.nanstd(v, ddof=1), 1, "std"), + (lambda v: np.nanvar(v, ddof=1), 1, "var"), + (np.nanmedian, 1, "median"), + ] +) +def base_functions(request): + """Fixture for base functions.""" + return request.param + + +@pytest.fixture( + params=[ + (np.max, None, "max"), + (np.min, None, "min"), + (np.sum, None, "sum"), + (np.mean, None, "mean"), + (lambda v: np.std(v, ddof=1), 1, "std"), + (lambda v: np.var(v, ddof=1), 1, "var"), + (np.median, None, "median"), + ] +) +def no_nan_functions(request): + """Fixture for no nan functions.""" + return request.param diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py index d20ab5131cf17..734bf7d99c5f5 100644 --- a/pandas/tests/window/moments/test_moments_expanding.py +++ b/pandas/tests/window/moments/test_moments_expanding.py @@ -145,50 +145,52 @@ def _check_expanding_has_min_periods(self, func, static_comp, has_min_periods): result = func(ser) tm.assert_almost_equal(result.iloc[-1], static_comp(ser[:50])) - @pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) - def test_expanding_apply_consistency(self, consistency_data, min_periods): - x, is_constant, no_nans = consistency_data - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message=".*(empty slice|0 for slice).*", - category=RuntimeWarning, + +@pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) +def test_expanding_apply_consistency( + consistency_data, base_functions, no_nan_functions, min_periods +): + x, is_constant, no_nans = consistency_data + functions = base_functions + + # GH 8269 + if no_nans: + functions = functions + no_nan_functions + f, require_min_periods, name = functions + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".*(empty slice|0 for slice).*", category=RuntimeWarning, + ) + # test consistency between expanding_xyz() and either (a) + # expanding_apply of Series.xyz(), or (b) expanding_apply of + # np.nanxyz() + expanding_f = getattr(x.expanding(min_periods=min_periods), name) + + if ( + require_min_periods + and (min_periods is not None) + and (min_periods < require_min_periods) + ): + continue + + if name == "count": + expanding_f_result = expanding_f() + expanding_apply_f_result = x.expanding(min_periods=0).apply( + func=f, raw=True ) - # test consistency between expanding_xyz() and either (a) - # expanding_apply of Series.xyz(), or (b) expanding_apply of - # np.nanxyz() - functions = self.base_functions - - # GH 8269 - if no_nans: - functions = self.base_functions + self.no_nan_functions - for (f, require_min_periods, name) in functions: - expanding_f = getattr(x.expanding(min_periods=min_periods), name) - - if ( - require_min_periods - and (min_periods is not None) - and (min_periods < require_min_periods) - ): - continue - - if name == "count": - expanding_f_result = expanding_f() - expanding_apply_f_result = x.expanding(min_periods=0).apply( - func=f, raw=True - ) - else: - if name in ["cov", "corr"]: - expanding_f_result = expanding_f(pairwise=False) - else: - expanding_f_result = expanding_f() - expanding_apply_f_result = x.expanding( - min_periods=min_periods - ).apply(func=f, raw=True) - - # GH 9422 - if name in ["sum", "prod"]: - tm.assert_equal(expanding_f_result, expanding_apply_f_result) + else: + if name in ["cov", "corr"]: + expanding_f_result = expanding_f(pairwise=False) + else: + expanding_f_result = expanding_f() + expanding_apply_f_result = x.expanding(min_periods=min_periods).apply( + func=f, raw=True + ) + + # GH 9422 + if name in ["sum", "prod"]: + tm.assert_equal(expanding_f_result, expanding_apply_f_result) @pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index c7d3a50ec13ae..62c6b278e6fa9 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -946,58 +946,6 @@ class TestRollingMomentsConsistency(ConsistencyBase): def setup_method(self, method): self._create_data() - @pytest.mark.parametrize( - "window,min_periods,center", list(_rolling_consistency_cases()) - ) - def test_rolling_apply_consistency( - self, consistency_data, window, min_periods, center - ): - x, is_constant, no_nans = consistency_data - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message=".*(empty slice|0 for slice).*", - category=RuntimeWarning, - ) - # test consistency between rolling_xyz() and either (a) - # rolling_apply of Series.xyz(), or (b) rolling_apply of - # np.nanxyz() - functions = self.base_functions - - # GH 8269 - if no_nans: - functions = self.base_functions + self.no_nan_functions - for (f, require_min_periods, name) in functions: - rolling_f = getattr( - x.rolling(window=window, center=center, min_periods=min_periods), - name, - ) - - if ( - require_min_periods - and (min_periods is not None) - and (min_periods < require_min_periods) - ): - continue - - if name == "count": - rolling_f_result = rolling_f() - rolling_apply_f_result = x.rolling( - window=window, min_periods=min_periods, center=center - ).apply(func=f, raw=True) - else: - if name in ["cov", "corr"]: - rolling_f_result = rolling_f(pairwise=False) - else: - rolling_f_result = rolling_f() - rolling_apply_f_result = x.rolling( - window=window, min_periods=min_periods, center=center - ).apply(func=f, raw=True) - - # GH 9422 - if name in ["sum", "prod"]: - tm.assert_equal(rolling_f_result, rolling_apply_f_result) - # binary moments def test_rolling_cov(self): A = self.series @@ -1052,6 +1000,57 @@ def test_flex_binary_frame(self, method): tm.assert_frame_equal(res3, exp) +@pytest.mark.parametrize( + "window,min_periods,center", list(_rolling_consistency_cases()) +) +def test_rolling_apply_consistency( + consistency_data, base_functions, no_nan_functions, window, min_periods, center +): + x, is_constant, no_nans = consistency_data + functions = base_functions + + # GH 8269 + if no_nans: + functions = functions + no_nan_functions + f, require_min_periods, name = functions + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".*(empty slice|0 for slice).*", category=RuntimeWarning, + ) + # test consistency between rolling_xyz() and either (a) + # rolling_apply of Series.xyz(), or (b) rolling_apply of + # np.nanxyz() + rolling_f = getattr( + x.rolling(window=window, center=center, min_periods=min_periods), name, + ) + + if ( + require_min_periods + and (min_periods is not None) + and (min_periods < require_min_periods) + ): + continue + + if name == "count": + rolling_f_result = rolling_f() + rolling_apply_f_result = x.rolling( + window=window, min_periods=min_periods, center=center + ).apply(func=f, raw=True) + else: + if name in ["cov", "corr"]: + rolling_f_result = rolling_f(pairwise=False) + else: + rolling_f_result = rolling_f() + rolling_apply_f_result = x.rolling( + window=window, min_periods=min_periods, center=center + ).apply(func=f, raw=True) + + # GH 9422 + if name in ["sum", "prod"]: + tm.assert_equal(rolling_f_result, rolling_apply_f_result) + + @pytest.mark.parametrize("window", range(7)) def test_rolling_corr_with_zero_variance(window): # GH 18430 From 494f94ceeecaa9e50e38093ec5935ccd2d6c2dcd Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sat, 2 May 2020 18:39:14 +0200 Subject: [PATCH 06/13] fixup --- pandas/tests/window/conftest.py | 83 ++++++++++--------- .../window/moments/test_moments_expanding.py | 62 +++++++------- .../window/moments/test_moments_rolling.py | 62 +++++++------- 3 files changed, 104 insertions(+), 103 deletions(-) diff --git a/pandas/tests/window/conftest.py b/pandas/tests/window/conftest.py index 074e397675e48..25fd829414fd7 100644 --- a/pandas/tests/window/conftest.py +++ b/pandas/tests/window/conftest.py @@ -244,48 +244,49 @@ def consistency_data(request): return request.param -@pytest.fixture( - params=[ - (lambda v: Series(v).count(), None, "count"), - (lambda v: Series(v).max(), None, "max"), - (lambda v: Series(v).min(), None, "min"), - (lambda v: Series(v).sum(), None, "sum"), - (lambda v: Series(v).mean(), None, "mean"), - (lambda v: Series(v).std(), 1, "std"), - (lambda v: Series(v).cov(Series(v)), None, "cov"), - (lambda v: Series(v).corr(Series(v)), None, "corr"), - (lambda v: Series(v).var(), 1, "var"), - # restore once GH 8086 is fixed - # lambda v: Series(v).skew(), 3, 'skew'), - # (lambda v: Series(v).kurt(), 4, 'kurt'), - # restore once GH 8084 is fixed - # lambda v: Series(v).quantile(0.3), None, 'quantile'), - (lambda v: Series(v).median(), None, "median"), - (np.nanmax, 1, "max"), - (np.nanmin, 1, "min"), - (np.nansum, 1, "sum"), - (np.nanmean, 1, "mean"), - (lambda v: np.nanstd(v, ddof=1), 1, "std"), - (lambda v: np.nanvar(v, ddof=1), 1, "var"), - (np.nanmedian, 1, "median"), - ] -) -def base_functions(request): +base_functions_list = [ + (lambda v: Series(v).count(), None, "count"), + (lambda v: Series(v).max(), None, "max"), + (lambda v: Series(v).min(), None, "min"), + (lambda v: Series(v).sum(), None, "sum"), + (lambda v: Series(v).mean(), None, "mean"), + (lambda v: Series(v).std(), 1, "std"), + (lambda v: Series(v).cov(Series(v)), None, "cov"), + (lambda v: Series(v).corr(Series(v)), None, "corr"), + (lambda v: Series(v).var(), 1, "var"), + # restore once GH 8086 is fixed + # lambda v: Series(v).skew(), 3, 'skew'), + # (lambda v: Series(v).kurt(), 4, 'kurt'), + # restore once GH 8084 is fixed + # lambda v: Series(v).quantile(0.3), None, 'quantile'), + (lambda v: Series(v).median(), None, "median"), + (np.nanmax, 1, "max"), + (np.nanmin, 1, "min"), + (np.nansum, 1, "sum"), + (np.nanmean, 1, "mean"), + (lambda v: np.nanstd(v, ddof=1), 1, "std"), + (lambda v: np.nanvar(v, ddof=1), 1, "var"), + (np.nanmedian, 1, "median"), +] + +no_nan_functions_list = [ + (np.max, None, "max"), + (np.min, None, "min"), + (np.sum, None, "sum"), + (np.mean, None, "mean"), + (lambda v: np.std(v, ddof=1), 1, "std"), + (lambda v: np.var(v, ddof=1), 1, "var"), + (np.median, None, "median"), +] + + +@pytest.fixture() +def base_functions(): """Fixture for base functions.""" - return request.param + return base_functions_list -@pytest.fixture( - params=[ - (np.max, None, "max"), - (np.min, None, "min"), - (np.sum, None, "sum"), - (np.mean, None, "mean"), - (lambda v: np.std(v, ddof=1), 1, "std"), - (lambda v: np.var(v, ddof=1), 1, "var"), - (np.median, None, "median"), - ] -) -def no_nan_functions(request): +@pytest.fixture() +def no_nan_functions(): """Fixture for no nan functions.""" - return request.param + return no_nan_functions_list diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py index 734bf7d99c5f5..aea181817efb6 100644 --- a/pandas/tests/window/moments/test_moments_expanding.py +++ b/pandas/tests/window/moments/test_moments_expanding.py @@ -151,12 +151,6 @@ def test_expanding_apply_consistency( consistency_data, base_functions, no_nan_functions, min_periods ): x, is_constant, no_nans = consistency_data - functions = base_functions - - # GH 8269 - if no_nans: - functions = functions + no_nan_functions - f, require_min_periods, name = functions with warnings.catch_warnings(): warnings.filterwarnings( @@ -165,32 +159,38 @@ def test_expanding_apply_consistency( # test consistency between expanding_xyz() and either (a) # expanding_apply of Series.xyz(), or (b) expanding_apply of # np.nanxyz() - expanding_f = getattr(x.expanding(min_periods=min_periods), name) - - if ( - require_min_periods - and (min_periods is not None) - and (min_periods < require_min_periods) - ): - continue - - if name == "count": - expanding_f_result = expanding_f() - expanding_apply_f_result = x.expanding(min_periods=0).apply( - func=f, raw=True - ) - else: - if name in ["cov", "corr"]: - expanding_f_result = expanding_f(pairwise=False) - else: + functions = base_functions + + # GH 8269 + if no_nans: + functions = no_nan_functions + base_functions + for (f, require_min_periods, name) in functions: + expanding_f = getattr(x.expanding(min_periods=min_periods), name) + + if ( + require_min_periods + and (min_periods is not None) + and (min_periods < require_min_periods) + ): + continue + + if name == "count": expanding_f_result = expanding_f() - expanding_apply_f_result = x.expanding(min_periods=min_periods).apply( - func=f, raw=True - ) - - # GH 9422 - if name in ["sum", "prod"]: - tm.assert_equal(expanding_f_result, expanding_apply_f_result) + expanding_apply_f_result = x.expanding(min_periods=0).apply( + func=f, raw=True + ) + else: + if name in ["cov", "corr"]: + expanding_f_result = expanding_f(pairwise=False) + else: + expanding_f_result = expanding_f() + expanding_apply_f_result = x.expanding(min_periods=min_periods).apply( + func=f, raw=True + ) + + # GH 9422 + if name in ["sum", "prod"]: + tm.assert_equal(expanding_f_result, expanding_apply_f_result) @pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 62c6b278e6fa9..6daabc6923a49 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1007,12 +1007,6 @@ def test_rolling_apply_consistency( consistency_data, base_functions, no_nan_functions, window, min_periods, center ): x, is_constant, no_nans = consistency_data - functions = base_functions - - # GH 8269 - if no_nans: - functions = functions + no_nan_functions - f, require_min_periods, name = functions with warnings.catch_warnings(): warnings.filterwarnings( @@ -1021,34 +1015,40 @@ def test_rolling_apply_consistency( # test consistency between rolling_xyz() and either (a) # rolling_apply of Series.xyz(), or (b) rolling_apply of # np.nanxyz() - rolling_f = getattr( - x.rolling(window=window, center=center, min_periods=min_periods), name, - ) + functions = base_functions + + # GH 8269 + if no_nans: + functions = no_nan_functions + base_functions + for (f, require_min_periods, name) in functions: + rolling_f = getattr( + x.rolling(window=window, center=center, min_periods=min_periods), name, + ) - if ( - require_min_periods - and (min_periods is not None) - and (min_periods < require_min_periods) - ): - continue + if ( + require_min_periods + and (min_periods is not None) + and (min_periods < require_min_periods) + ): + pass - if name == "count": - rolling_f_result = rolling_f() - rolling_apply_f_result = x.rolling( - window=window, min_periods=min_periods, center=center - ).apply(func=f, raw=True) - else: - if name in ["cov", "corr"]: - rolling_f_result = rolling_f(pairwise=False) - else: + if name == "count": rolling_f_result = rolling_f() - rolling_apply_f_result = x.rolling( - window=window, min_periods=min_periods, center=center - ).apply(func=f, raw=True) - - # GH 9422 - if name in ["sum", "prod"]: - tm.assert_equal(rolling_f_result, rolling_apply_f_result) + rolling_apply_f_result = x.rolling( + window=window, min_periods=min_periods, center=center + ).apply(func=f, raw=True) + else: + if name in ["cov", "corr"]: + rolling_f_result = rolling_f(pairwise=False) + else: + rolling_f_result = rolling_f() + rolling_apply_f_result = x.rolling( + window=window, min_periods=min_periods, center=center + ).apply(func=f, raw=True) + + # GH 9422 + if name in ["sum", "prod"]: + tm.assert_equal(rolling_f_result, rolling_apply_f_result) @pytest.mark.parametrize("window", range(7)) From 49185528f8a21a9799b1f38144da007c00f14b6c Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 3 May 2020 09:33:12 +0200 Subject: [PATCH 07/13] mark consistency cov as slow --- pandas/tests/window/moments/test_moments_expanding.py | 1 + pandas/tests/window/moments/test_moments_rolling.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py index aea181817efb6..1d1c8078814b5 100644 --- a/pandas/tests/window/moments/test_moments_expanding.py +++ b/pandas/tests/window/moments/test_moments_expanding.py @@ -219,6 +219,7 @@ def test_expanding_consistency_std(consistency_data, min_periods): ) +@pytest.mark.slow @pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) def test_expanding_consistency_cov(consistency_data, min_periods): x, is_constant, no_nans = consistency_data diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 6daabc6923a49..65901227534d3 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1476,6 +1476,7 @@ def test_rolling_consistency_std(consistency_data, window, min_periods, center): ) +@pytest.mark.slow @pytest.mark.parametrize( "window,min_periods,center", list(_rolling_consistency_cases()) ) From 6bf8c7f21cb3808614801149dcee3a078153a341 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 3 May 2020 09:43:17 +0200 Subject: [PATCH 08/13] remove slow for exapnding --- pandas/tests/window/moments/test_moments_expanding.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py index 1d1c8078814b5..aea181817efb6 100644 --- a/pandas/tests/window/moments/test_moments_expanding.py +++ b/pandas/tests/window/moments/test_moments_expanding.py @@ -219,7 +219,6 @@ def test_expanding_consistency_std(consistency_data, min_periods): ) -@pytest.mark.slow @pytest.mark.parametrize("min_periods", [0, 1, 2, 3, 4]) def test_expanding_consistency_cov(consistency_data, min_periods): x, is_constant, no_nans = consistency_data From 88d8439a07ac2360eb2f7a0d7522b9cee22c1046 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 3 May 2020 10:41:52 +0200 Subject: [PATCH 09/13] fixup --- pandas/tests/window/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/window/conftest.py b/pandas/tests/window/conftest.py index 25fd829414fd7..abdbfd10ecfd5 100644 --- a/pandas/tests/window/conftest.py +++ b/pandas/tests/window/conftest.py @@ -280,13 +280,13 @@ def consistency_data(request): ] -@pytest.fixture() +@pytest.fixture def base_functions(): """Fixture for base functions.""" return base_functions_list -@pytest.fixture() +@pytest.fixture def no_nan_functions(): """Fixture for no nan functions.""" return no_nan_functions_list From a8c9bac6265f89e615cabf2b8b6043abebdf33c6 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 3 May 2020 17:40:14 +0200 Subject: [PATCH 10/13] run again --- pandas/tests/window/conftest.py | 48 ------------------------- pandas/tests/window/moments/conftest.py | 48 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pandas/tests/window/conftest.py b/pandas/tests/window/conftest.py index abdbfd10ecfd5..856c8f3884816 100644 --- a/pandas/tests/window/conftest.py +++ b/pandas/tests/window/conftest.py @@ -242,51 +242,3 @@ def no_nans(x): def consistency_data(request): """Create consistency data""" return request.param - - -base_functions_list = [ - (lambda v: Series(v).count(), None, "count"), - (lambda v: Series(v).max(), None, "max"), - (lambda v: Series(v).min(), None, "min"), - (lambda v: Series(v).sum(), None, "sum"), - (lambda v: Series(v).mean(), None, "mean"), - (lambda v: Series(v).std(), 1, "std"), - (lambda v: Series(v).cov(Series(v)), None, "cov"), - (lambda v: Series(v).corr(Series(v)), None, "corr"), - (lambda v: Series(v).var(), 1, "var"), - # restore once GH 8086 is fixed - # lambda v: Series(v).skew(), 3, 'skew'), - # (lambda v: Series(v).kurt(), 4, 'kurt'), - # restore once GH 8084 is fixed - # lambda v: Series(v).quantile(0.3), None, 'quantile'), - (lambda v: Series(v).median(), None, "median"), - (np.nanmax, 1, "max"), - (np.nanmin, 1, "min"), - (np.nansum, 1, "sum"), - (np.nanmean, 1, "mean"), - (lambda v: np.nanstd(v, ddof=1), 1, "std"), - (lambda v: np.nanvar(v, ddof=1), 1, "var"), - (np.nanmedian, 1, "median"), -] - -no_nan_functions_list = [ - (np.max, None, "max"), - (np.min, None, "min"), - (np.sum, None, "sum"), - (np.mean, None, "mean"), - (lambda v: np.std(v, ddof=1), 1, "std"), - (lambda v: np.var(v, ddof=1), 1, "var"), - (np.median, None, "median"), -] - - -@pytest.fixture -def base_functions(): - """Fixture for base functions.""" - return base_functions_list - - -@pytest.fixture -def no_nan_functions(): - """Fixture for no nan functions.""" - return no_nan_functions_list diff --git a/pandas/tests/window/moments/conftest.py b/pandas/tests/window/moments/conftest.py index 2002f4d0bff43..a6eaf184154f5 100644 --- a/pandas/tests/window/moments/conftest.py +++ b/pandas/tests/window/moments/conftest.py @@ -18,3 +18,51 @@ def binary_ew_data(): @pytest.fixture(params=[0, 1, 2]) def min_periods(request): return request.param + + +base_functions_list = [ + (lambda v: Series(v).count(), None, "count"), + (lambda v: Series(v).max(), None, "max"), + (lambda v: Series(v).min(), None, "min"), + (lambda v: Series(v).sum(), None, "sum"), + (lambda v: Series(v).mean(), None, "mean"), + (lambda v: Series(v).std(), 1, "std"), + (lambda v: Series(v).cov(Series(v)), None, "cov"), + (lambda v: Series(v).corr(Series(v)), None, "corr"), + (lambda v: Series(v).var(), 1, "var"), + # restore once GH 8086 is fixed + # lambda v: Series(v).skew(), 3, 'skew'), + # (lambda v: Series(v).kurt(), 4, 'kurt'), + # restore once GH 8084 is fixed + # lambda v: Series(v).quantile(0.3), None, 'quantile'), + (lambda v: Series(v).median(), None, "median"), + (np.nanmax, 1, "max"), + (np.nanmin, 1, "min"), + (np.nansum, 1, "sum"), + (np.nanmean, 1, "mean"), + (lambda v: np.nanstd(v, ddof=1), 1, "std"), + (lambda v: np.nanvar(v, ddof=1), 1, "var"), + (np.nanmedian, 1, "median"), +] + +no_nan_functions_list = [ + (np.max, None, "max"), + (np.min, None, "min"), + (np.sum, None, "sum"), + (np.mean, None, "mean"), + (lambda v: np.std(v, ddof=1), 1, "std"), + (lambda v: np.var(v, ddof=1), 1, "var"), + (np.median, None, "median"), +] + + +@pytest.fixture(scope="session") +def base_functions(): + """Fixture for base functions.""" + return base_functions_list + + +@pytest.fixture(scope="session") +def no_nan_functions(): + """Fixture for no nan functions.""" + return no_nan_functions_list From ddc816007a8da15e763b431ee89389ec37026152 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 3 May 2020 21:17:37 +0200 Subject: [PATCH 11/13] run again --- pandas/tests/window/moments/test_moments_expanding.py | 2 +- pandas/tests/window/moments/test_moments_rolling.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py index aea181817efb6..6aff5b78b4ada 100644 --- a/pandas/tests/window/moments/test_moments_expanding.py +++ b/pandas/tests/window/moments/test_moments_expanding.py @@ -163,7 +163,7 @@ def test_expanding_apply_consistency( # GH 8269 if no_nans: - functions = no_nan_functions + base_functions + functions = base_functions + no_nan_functions for (f, require_min_periods, name) in functions: expanding_f = getattr(x.expanding(min_periods=min_periods), name) diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 65901227534d3..074c2736b7ab8 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1000,6 +1000,7 @@ def test_flex_binary_frame(self, method): tm.assert_frame_equal(res3, exp) +@pytest.mark.slow @pytest.mark.parametrize( "window,min_periods,center", list(_rolling_consistency_cases()) ) @@ -1030,7 +1031,7 @@ def test_rolling_apply_consistency( and (min_periods is not None) and (min_periods < require_min_periods) ): - pass + continue if name == "count": rolling_f_result = rolling_f() From a602629ee81eab6765d51c57edfddf74bae709c8 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Wed, 6 May 2020 14:49:44 +0200 Subject: [PATCH 12/13] add comments --- pandas/tests/window/moments/conftest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pandas/tests/window/moments/conftest.py b/pandas/tests/window/moments/conftest.py index a6eaf184154f5..39e6ae71162a9 100644 --- a/pandas/tests/window/moments/conftest.py +++ b/pandas/tests/window/moments/conftest.py @@ -58,11 +58,21 @@ def min_periods(request): @pytest.fixture(scope="session") def base_functions(): - """Fixture for base functions.""" + """Fixture for base functions. + + Returns + ------- + List of tuples: (applied function, require_min_periods, name of applied function) + """ return base_functions_list @pytest.fixture(scope="session") def no_nan_functions(): - """Fixture for no nan functions.""" + """Fixture for no nan functions. + + Returns + ------- + List of tuples: (applied function, require_min_periods, name of applied function) + """ return no_nan_functions_list From 2a485e1afe9ece9120ebfe6c85e23a564b87db07 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Wed, 6 May 2020 20:11:36 +0200 Subject: [PATCH 13/13] mark slow --- pandas/tests/window/moments/test_moments_rolling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 074c2736b7ab8..28ad2b8663efe 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -1431,6 +1431,7 @@ def test_moment_functions_zero_length_pairwise(): tm.assert_frame_equal(df2_result, df2_expected) +@pytest.mark.slow @pytest.mark.parametrize( "window,min_periods,center", list(_rolling_consistency_cases()) ) @@ -1455,6 +1456,7 @@ def test_rolling_consistency_var(consistency_data, window, min_periods, center): ) +@pytest.mark.slow @pytest.mark.parametrize( "window,min_periods,center", list(_rolling_consistency_cases()) )