From 6b88dcbc957df7e7b757050e573ed54f2c950bbd Mon Sep 17 00:00:00 2001 From: pilkibun Date: Thu, 13 Jun 2019 23:22:37 +0300 Subject: [PATCH 1/9] TST: introduce test decorator skip_if_np_lt(ver_string) --- pandas/compat/numpy/__init__.py | 5 +++-- pandas/tests/arrays/sparse/test_array.py | 9 ++++----- pandas/tests/frame/test_analytics.py | 16 ++++++++-------- pandas/tests/series/test_analytics.py | 6 +++--- pandas/util/_test_decorators.py | 12 +++++++++--- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/pandas/compat/numpy/__init__.py b/pandas/compat/numpy/__init__.py index 3499d631376d8..c738cc74e46a4 100644 --- a/pandas/compat/numpy/__init__.py +++ b/pandas/compat/numpy/__init__.py @@ -1,9 +1,9 @@ """ support numpy compatiblitiy across versions """ -import re -import numpy as np from distutils.version import LooseVersion +import re +import numpy as np # numpy versioning _np_version = np.__version__ @@ -62,6 +62,7 @@ def np_array_datetime64_compat(arr, *args, **kwargs): __all__ = ['np', + '_np_version', '_np_version_under1p14', '_np_version_under1p15', '_np_version_under1p16', diff --git a/pandas/tests/arrays/sparse/test_array.py b/pandas/tests/arrays/sparse/test_array.py index 659f2b97485a9..c0a1b32079044 100644 --- a/pandas/tests/arrays/sparse/test_array.py +++ b/pandas/tests/arrays/sparse/test_array.py @@ -6,7 +6,6 @@ import pytest from pandas._libs.sparse import IntIndex -from pandas.compat.numpy import _np_version_under1p16 import pandas.util._test_decorators as td import pandas as pd @@ -175,8 +174,8 @@ def test_constructor_inferred_fill_value(self, data, fill_value): @pytest.mark.parametrize('format', ['coo', 'csc', 'csr']) @pytest.mark.parametrize('size', [ pytest.param(0, - marks=pytest.mark.skipif(_np_version_under1p16, - reason='NumPy-11383')), + marks=td.skip_if_np_lt("1.16", + reason='NumPy-11383')), 10 ]) @td.skip_if_no_scipy @@ -870,7 +869,7 @@ def test_all(self, data, pos, neg): ([1, 2, 1], 1, 0), ([1.0, 2.0, 1.0], 1.0, 0.0) ]) - @td.skip_if_np_lt_115 # prior didn't dispatch + @td.skip_if_np_lt("1.15") # prior didn't dispatch def test_numpy_all(self, data, pos, neg): # GH 17570 out = np.all(SparseArray(data)) @@ -916,7 +915,7 @@ def test_any(self, data, pos, neg): ([0, 2, 0], 2, 0), ([0.0, 2.0, 0.0], 2.0, 0.0) ]) - @td.skip_if_np_lt_115 # prior didn't dispatch + @td.skip_if_np_lt("1.15") # prior didn't dispatch def test_numpy_any(self, data, pos, neg): # GH 17570 out = np.any(SparseArray(data)) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 18d8d351e48c1..01a398584b5e1 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1565,21 +1565,21 @@ def test_any_all_bool_only(self): (np.all, {'A': pd.Series([0, 1], dtype=int)}, False), (np.any, {'A': pd.Series([0, 1], dtype=int)}, True), pytest.param(np.all, {'A': pd.Series([0, 1], dtype='M8[ns]')}, False, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.any, {'A': pd.Series([0, 1], dtype='M8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.all, {'A': pd.Series([1, 2], dtype='M8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.any, {'A': pd.Series([1, 2], dtype='M8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.all, {'A': pd.Series([0, 1], dtype='m8[ns]')}, False, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.any, {'A': pd.Series([0, 1], dtype='m8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.all, {'A': pd.Series([1, 2], dtype='m8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), pytest.param(np.any, {'A': pd.Series([1, 2], dtype='m8[ns]')}, True, - marks=[td.skip_if_np_lt_115]), + marks=[td.skip_if_np_lt("1.15")]), (np.all, {'A': pd.Series([0, 1], dtype='category')}, False), (np.any, {'A': pd.Series([0, 1], dtype='category')}, True), (np.all, {'A': pd.Series([1, 2], dtype='category')}, True), diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index e5eb7d19dc649..aed08b78fe640 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1105,7 +1105,7 @@ def test_value_counts_categorical_not_ordered(self): dict(keepdims=True), dict(out=object()), ]) - @td.skip_if_np_lt_115 + @td.skip_if_np_lt("1.15") def test_validate_any_all_out_keepdims_raises(self, kwargs, func): s = pd.Series([1, 2]) param = list(kwargs)[0] @@ -1117,7 +1117,7 @@ def test_validate_any_all_out_keepdims_raises(self, kwargs, func): with pytest.raises(ValueError, match=msg): func(s, **kwargs) - @td.skip_if_np_lt_115 + @td.skip_if_np_lt("1.15") def test_validate_sum_initial(self): s = pd.Series([1, 2]) msg = (r"the 'initial' parameter is not " @@ -1136,7 +1136,7 @@ def test_validate_median_initial(self): # method instead of the ufunc. s.median(overwrite_input=True) - @td.skip_if_np_lt_115 + @td.skip_if_np_lt("1.15") def test_validate_stat_keepdims(self): s = pd.Series([1, 2]) msg = (r"the 'keepdims' parameter is not " diff --git a/pandas/util/_test_decorators.py b/pandas/util/_test_decorators.py index 4cc316ffdd7ab..0cb82c0028c90 100644 --- a/pandas/util/_test_decorators.py +++ b/pandas/util/_test_decorators.py @@ -23,6 +23,7 @@ def test_foo(): For more information, refer to the ``pytest`` documentation on ``skipif``. """ +from distutils.version import LooseVersion import locale from typing import Optional @@ -30,7 +31,7 @@ def test_foo(): import pytest from pandas.compat import is_platform_32bit, is_platform_windows -from pandas.compat.numpy import _np_version_under1p15 +from pandas.compat.numpy import _np_version from pandas.core.computation.expressions import ( _NUMEXPR_INSTALLED, _USE_NUMEXPR) @@ -142,8 +143,6 @@ def skip_if_no( skip_if_no_mpl = pytest.mark.skipif(_skip_if_no_mpl(), reason="Missing matplotlib dependency") -skip_if_np_lt_115 = pytest.mark.skipif(_np_version_under1p15, - reason="NumPy 1.15 or greater required") skip_if_mpl = pytest.mark.skipif(not _skip_if_no_mpl(), reason="matplotlib is present") skip_if_32bit = pytest.mark.skipif(is_platform_32bit(), @@ -168,6 +167,13 @@ def skip_if_no( installed=_NUMEXPR_INSTALLED)) +def skip_if_np_lt(ver_str, reason=None, *args, **kwds): + if reason is None: + reason = "NumPy %s or greater required" % ver_str + return pytest.mark.skipif(_np_version < LooseVersion(ver_str), + reason=reason, *args, **kwds) + + def parametrize_fixture_doc(*args): """ Intended for use as a decorator for parametrized fixture, From 6c2ca298de5a959ab0afceaee31806d077248b30 Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:33:48 +0300 Subject: [PATCH 2/9] TST: add td.skip_if_no_NEP18 --- pandas/compat/numpy/__init__.py | 5 ++++- pandas/util/_test_decorators.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pandas/compat/numpy/__init__.py b/pandas/compat/numpy/__init__.py index c738cc74e46a4..44ac25ea2f57c 100644 --- a/pandas/compat/numpy/__init__.py +++ b/pandas/compat/numpy/__init__.py @@ -13,7 +13,10 @@ _np_version_under1p16 = _nlv < LooseVersion('1.16') _np_version_under1p17 = _nlv < LooseVersion('1.17') _is_numpy_dev = '.dev' in str(_nlv) - +try: + _NEP18_enabled = np.core.overrides.ENABLE_ARRAY_FUNCTION +except Exception: + _NEP18_enabled = False if _nlv < '1.13.3': raise ImportError('this version of pandas is incompatible with ' diff --git a/pandas/util/_test_decorators.py b/pandas/util/_test_decorators.py index 0cb82c0028c90..690df6677b1c2 100644 --- a/pandas/util/_test_decorators.py +++ b/pandas/util/_test_decorators.py @@ -31,7 +31,7 @@ def test_foo(): import pytest from pandas.compat import is_platform_32bit, is_platform_windows -from pandas.compat.numpy import _np_version +from pandas.compat.numpy import _NEP18_enabled, _np_version from pandas.core.computation.expressions import ( _NUMEXPR_INSTALLED, _USE_NUMEXPR) @@ -165,6 +165,8 @@ def skip_if_no( "installed->{installed}".format( enabled=_USE_NUMEXPR, installed=_NUMEXPR_INSTALLED)) +skip_if_no_NEP18 = pytest.mark.skipif(not _NEP18_enabled, + reason="numpy NEP-18 disabled") def skip_if_np_lt(ver_str, reason=None, *args, **kwds): From 0a8ef62f441da73fe63990820660b256d9080df0 Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:34:23 +0300 Subject: [PATCH 3/9] TST: add tests for DecimalArray round and sum --- .../tests/extension/decimal/test_decimal.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index 97fae41bcc720..66f2e013d3e4b 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -5,6 +5,8 @@ import numpy as np import pytest +import pandas.util._test_decorators as td + import pandas as pd from pandas.tests.extension import base import pandas.util.testing as tm @@ -380,6 +382,26 @@ def test_divmod_array(reverse, expected_div, expected_mod): tm.assert_extension_array_equal(mod, expected_mod) +# numpy 1.17 has NEP-18 on by default +# for numpy 1.16 set shell variable with +# "export NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1" +# before running pytest/python. +# verify by checking value of `np.core.overrides.ENABLE_ARRAY_FUNCTION` +@td.skip_if_no_NEP18 +def test_series_round(): + ser = pd.Series(to_decimal([1.1, 2.4, 3.1])).round() + expected = pd.Series(to_decimal([1, 2, 3])) + tm.assert_extension_array_equal(ser.array, expected.array) + tm.assert_series_equal(ser, expected) + + +@td.skip_if_no_NEP18 +def test_series_round_then_sum(): + result = pd.Series(to_decimal([1.1, 2.4, 3.1])).round().sum(skipna=False) + expected = decimal.Decimal("6") + assert result == expected + + def test_formatting_values_deprecated(): class DecimalArray2(DecimalArray): def _formatting_values(self): From f038c18ff3aa8e05e5ffa8e6423407fcbe06b5f5 Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:35:36 +0300 Subject: [PATCH 4/9] CLN: In DecimalArray make aliases as properties, _data may be assigned to --- pandas/tests/extension/decimal/array.py | 26 +++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 1823eeb4d7fc0..80d57569f3626 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -59,14 +59,28 @@ def __init__(self, values, dtype=None, copy=False, context=None): values = np.asarray(values, dtype=object) self._data = values - # Some aliases for common attribute names to ensure pandas supports - # these - self._items = self.data = self._data - # those aliases are currently not working due to assumptions - # in internal code (GH-20735) - # self._values = self.values = self.data self._dtype = DecimalDtype(context) + # aliases for common attribute names, to ensure pandas supports these + @property + def _items(self): + return self._data + + @property + def data(self): + return self._data + + # those aliases are currently not working due to assumptions + # in internal code (GH-20735) + # @property + # def _values(self): + # return self._data + + # @property + # def values(self): + # return self._data + # end aliases + @property def dtype(self): return self._dtype From 068d8eea1793b67caa8be2b47037067570d213f0 Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:36:17 +0300 Subject: [PATCH 5/9] CLN: tweak brittle error string check --- pandas/tests/extension/base/methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/extension/base/methods.py b/pandas/tests/extension/base/methods.py index 1852edaa9e748..82c6ce9ab137c 100644 --- a/pandas/tests/extension/base/methods.py +++ b/pandas/tests/extension/base/methods.py @@ -323,7 +323,7 @@ def test_repeat(self, data, repeats, as_series, use_numpy): self.assert_equal(result, expected) @pytest.mark.parametrize('repeats, kwargs, error, msg', [ - (2, dict(axis=1), ValueError, "'axis"), + (2, dict(axis=1), ValueError, "'?axis"), (-1, dict(), ValueError, "negative"), ([1, 2], dict(), ValueError, "shape"), (2, dict(foo='bar'), TypeError, "'foo'")]) From 5af678a334972a738f25c92808da6a55acfd84fa Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:36:42 +0300 Subject: [PATCH 6/9] ENH: implement NEP-18 handler for DecimalArray --- pandas/tests/extension/decimal/array.py | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 80d57569f3626..23ddfca043452 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -11,6 +11,8 @@ from pandas.api.extensions import register_extension_dtype from pandas.core.arrays import ExtensionArray, ExtensionScalarOpsMixin +_should_cast_results = [np.repeat] + @register_extension_dtype class DecimalDtype(ExtensionDtype): @@ -167,6 +169,40 @@ def _reduce(self, name, skipna=True, **kwargs): "the {} operation".format(name)) return op(axis=0) + # numpy experimental NEP-18 (opt-in numpy 1.16, enabled in in 1.17) + def __array_function__(self, func, types, args, kwargs): + def coerce_EA(coll): + # In order to delegate to numpy, we have to coerce any + # ExtensionArrays to the best numpy-friendly dtype approximation + # Different functions take different arguments, which may be + # nested collections, so we look at everything. Sigh. + for i in range(len(coll)): + if isinstance(coll[i], (tuple, list)): + coll[i] = coerce_EA(list(coll[i])) + else: + if isinstance(coll[i], DecimalArray): + # TODO: how to check for any ndarray-like with + # non-numpy dtype? + coll[i] = np.array(coll[i], dtype=object) + + return coll + + if func is np.round_: + values = [decimal.Decimal(round(_)) + for _ in self._data] + return DecimalArray(values, dtype=self.dtype) + + elif True: # just assume we can handle all functions + args = coerce_EA(list(args)) + result = func(*args, **kwargs) + + if func in _should_cast_results: + result = pd.array(result, dtype=self.dtype) + + return result + else: + return NotImplemented + def to_decimal(values, context=None): return DecimalArray([decimal.Decimal(x) for x in values], context=context) From 85993a22884f90786486fff4e5e649328576dbce Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sat, 15 Jun 2019 07:37:19 +0300 Subject: [PATCH 7/9] ENH: Prepare for EA support in series.round() --- pandas/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index f0362596920a6..b7c8e83617b89 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -2098,7 +2098,7 @@ def round(self, decimals=0, *args, **kwargs): dtype: float64 """ nv.validate_round(args, kwargs) - result = com.values_from_object(self).round(decimals) + result = np.round(self.array, decimals=decimals) result = self._constructor(result, index=self.index).__finalize__(self) return result From a527131e32749a64bd47054d5859c9367f2243ea Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sun, 16 Jun 2019 23:34:19 +0300 Subject: [PATCH 8/9] Revert "CLN: In DecimalArray make aliases as properties, _data may be assigned to" This reverts commit f038c18ff3aa8e05e5ffa8e6423407fcbe06b5f5. --- pandas/tests/extension/decimal/array.py | 26 ++++++------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 23ddfca043452..2a95fdbb76fda 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -61,28 +61,14 @@ def __init__(self, values, dtype=None, copy=False, context=None): values = np.asarray(values, dtype=object) self._data = values + # Some aliases for common attribute names to ensure pandas supports + # these + self._items = self.data = self._data + # those aliases are currently not working due to assumptions + # in internal code (GH-20735) + # self._values = self.values = self.data self._dtype = DecimalDtype(context) - # aliases for common attribute names, to ensure pandas supports these - @property - def _items(self): - return self._data - - @property - def data(self): - return self._data - - # those aliases are currently not working due to assumptions - # in internal code (GH-20735) - # @property - # def _values(self): - # return self._data - - # @property - # def values(self): - # return self._data - # end aliases - @property def dtype(self): return self._dtype From ffabac2be8f4b8baa7587011d1f98362de826531 Mon Sep 17 00:00:00 2001 From: pilkibun Date: Sun, 16 Jun 2019 23:35:59 +0300 Subject: [PATCH 9/9] Revert "CLN: tweak brittle error string check" This reverts commit 068d8eea1793b67caa8be2b47037067570d213f0. --- pandas/tests/extension/base/methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/extension/base/methods.py b/pandas/tests/extension/base/methods.py index 82c6ce9ab137c..1852edaa9e748 100644 --- a/pandas/tests/extension/base/methods.py +++ b/pandas/tests/extension/base/methods.py @@ -323,7 +323,7 @@ def test_repeat(self, data, repeats, as_series, use_numpy): self.assert_equal(result, expected) @pytest.mark.parametrize('repeats, kwargs, error, msg', [ - (2, dict(axis=1), ValueError, "'?axis"), + (2, dict(axis=1), ValueError, "'axis"), (-1, dict(), ValueError, "negative"), ([1, 2], dict(), ValueError, "shape"), (2, dict(foo='bar'), TypeError, "'foo'")])