From c3798d2b9fb4a0139afec791babce866584b2d92 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Mon, 4 Apr 2022 11:08:31 +0100 Subject: [PATCH 01/16] Reintroduce `inspect` use for sig tests --- array_api_tests/test_signatures.py | 392 ++++++++++------------------- 1 file changed, 128 insertions(+), 264 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 2e197ee9..a3253bb7 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -1,282 +1,146 @@ -import inspect -from itertools import chain +from inspect import Parameter, Signature, signature +from types import FunctionType +from typing import Callable, Dict import pytest +from hypothesis import given -from ._array_module import mod, mod_name, ones, eye, float64, bool, int64, _UndefinedStub -from .pytest_helpers import raises, doesnt_raise -from . import dtype_helpers as dh +from . import hypothesis_helpers as hh +from . import xps +from ._array_module import mod as xp +from .stubs import array_methods, category_to_funcs, extension_to_funcs -from . import stubs +pytestmark = pytest.mark.ci +kind_to_str: Dict[Parameter, str] = { + Parameter.POSITIONAL_OR_KEYWORD: "normal argument", + Parameter.POSITIONAL_ONLY: "pos-only argument", + Parameter.KEYWORD_ONLY: "keyword-only argument", + Parameter.VAR_POSITIONAL: "star-args (i.e. *args) argument", + Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument", +} -def extension_module(name) -> bool: - for funcs in stubs.extension_to_funcs.values(): - for func in funcs: - if name == func.__name__: - return True - else: - return False - - -params = [] -for name in [f.__name__ for funcs in stubs.category_to_funcs.values() for f in funcs]: - if name in ["where", "expand_dims", "reshape"]: - params.append(pytest.param(name, marks=pytest.mark.skip(reason="faulty test"))) - else: - params.append(name) +def _test_signature( + func: Callable, stub: FunctionType, ignore_first_stub_param: bool = False +): + """ + Signature of function is correct enough to not affect interoperability -for ext, name in [(ext, f.__name__) for ext, funcs in stubs.extension_to_funcs.items() for f in funcs]: - params.append(pytest.param(name, marks=pytest.mark.xp_extension(ext))) + We're not interested in being 100% strict - instead we focus on areas which + could affect interop, e.g. with + def add(x1, x2, /): + ... -def array_method(name) -> bool: - return name in [f.__name__ for f in stubs.array_methods] + x1 and x2 don't need to be pos-only for the purposes of interoperability, but with -def function_category(name) -> str: - for category, funcs in chain(stubs.category_to_funcs.items(), stubs.extension_to_funcs.items()): - for func in funcs: - if name == func.__name__: - return category + def squeeze(x, /, axis): + ... -def example_argument(arg, func_name, dtype): - """ - Get an example argument for the argument arg for the function func_name + axis has to be pos-or-keyword to support both styles - The full tests for function behavior is in other files. We just need to - have an example input for each argument name that should work so that we - can check if the argument is implemented at all. + >>> squeeze(x, 0) + ... + >>> squeeze(x, axis=0) + ... """ - # Note: for keyword arguments that have a default, this should be - # different from the default, as the default argument is tested separately - # (it can have the same behavior as the default, just not literally the - # same value). - known_args = dict( - api_version='2021.1', - arrays=(ones((1, 3, 3), dtype=dtype), ones((1, 3, 3), dtype=dtype)), - # These cannot be the same as each other, which is why all our test - # arrays have to have at least 3 dimensions. - axis1=2, - axis2=2, - axis=1, - axes=(2, 1, 0), - copy=True, - correction=1.0, - descending=True, - # TODO: This will only work on the NumPy implementation. The exact - # value of the device keyword will vary across implementations, so we - # need some way to infer it or for libraries to specify a list of - # valid devices. - device='cpu', - dtype=float64, - endpoint=False, - fill_value=1.0, - from_=int64, - full_matrices=False, - k=1, - keepdims=True, - key=(0, 0), - indexing='ij', - mode='complete', - n=2, - n_cols=1, - n_rows=1, - num=2, - offset=1, - ord=1, - obj = [[[1, 1, 1], [1, 1, 1], [1, 1, 1]]], - other=ones((3, 3), dtype=dtype), - return_counts=True, - return_index=True, - return_inverse=True, - rtol=1e-10, - self=ones((3, 3), dtype=dtype), - shape=(1, 3, 3), - shift=1, - sorted=False, - stable=False, - start=0, - step=2, - stop=1, - # TODO: Update this to be non-default. See the comment on "device" above. - stream=None, - to=float64, - type=float64, - upper=True, - value=0, - x1=ones((1, 3, 3), dtype=dtype), - x2=ones((1, 3, 3), dtype=dtype), - x=ones((1, 3, 3), dtype=dtype), - ) - if not isinstance(bool, _UndefinedStub): - known_args['condition'] = ones((1, 3, 3), dtype=bool), - - if arg in known_args: - # Special cases: - - # squeeze() requires an axis of size 1, but other functions such as - # cross() require axes of size >1 - if func_name == 'squeeze' and arg == 'axis': - return 0 - # ones() is not invertible - # finfo requires a float dtype and iinfo requires an int dtype - elif func_name == 'iinfo' and arg == 'type': - return int64 - # tensordot args must be contractible with each other - elif func_name == 'tensordot' and arg == 'x2': - return ones((3, 3, 1), dtype=dtype) - # tensordot "axes" is either a number representing the number of - # contractible axes or a 2-tuple or axes - elif func_name == 'tensordot' and arg == 'axes': - return 1 - # The inputs to outer() must be 1-dimensional - elif func_name == 'outer' and arg in ['x1', 'x2']: - return ones((3,), dtype=dtype) - # Linear algebra functions tend to error if the input isn't "nice" as - # a matrix - elif arg.startswith('x') and func_name in [f.__name__ for f in stubs.extension_to_funcs["linalg"]]: - return eye(3) - return known_args[arg] - else: - raise RuntimeError(f"Don't know how to test argument {arg}. Please update test_signatures.py") - -@pytest.mark.parametrize('name', params) -def test_has_names(name): - if extension_module(name): - ext = next( - ext for ext, funcs in stubs.extension_to_funcs.items() - if name in [f.__name__ for f in funcs] + try: + sig = signature(func) + except ValueError: + pytest.skip( + msg=f"type({stub.__name__})={type(func)} not supported by inspect.signature()" + ) + params = list(sig.parameters.values()) + + stub_sig = signature(stub) + stub_params = list(stub_sig.parameters.values()) + if ignore_first_stub_param: + stub_params = stub_params[1:] + stub = Signature( + parameters=stub_params, return_annotation=stub_sig.return_annotation ) - ext_mod = getattr(mod, ext) - assert hasattr(ext_mod, name), f"{mod_name} is missing the {function_category(name)} extension function {name}()" - elif array_method(name): - arr = ones((1, 1)) - if name not in [f.__name__ for f in stubs.array_methods]: - assert hasattr(arr, name), f"The array object is missing the attribute {name}" - else: - assert hasattr(arr, name), f"The array object is missing the method {name}()" - else: - assert hasattr(mod, name), f"{mod_name} is missing the {function_category(name)} function {name}()" - -@pytest.mark.parametrize('name', params) -def test_function_positional_args(name): - # Note: We can't actually test that positional arguments are - # positional-only, as that would require knowing the argument name and - # checking that it can't be used as a keyword argument. But argument name - # inspection does not work for most array library functions that are not - # written in pure Python (e.g., it won't work for numpy ufuncs). - - if extension_module(name): - return - - dtype = None - if (name.startswith('__i') and name not in ['__int__', '__invert__', '__index__'] - or name.startswith('__r') and name != '__rshift__'): - n = f'__{name[3:]}' - else: - n = name - in_dtypes = dh.func_in_dtypes.get(n, dh.float_dtypes) - if bool in in_dtypes: - dtype = bool - elif all(d in in_dtypes for d in dh.all_int_dtypes): - dtype = int64 - - if array_method(name): - if name == '__bool__': - _mod = ones((), dtype=bool) - elif name in ['__int__', '__index__']: - _mod = ones((), dtype=int64) - elif name == '__float__': - _mod = ones((), dtype=float64) - else: - _mod = example_argument('self', name, dtype) - elif '.' in name: - extension_module_name, name = name.split('.') - _mod = getattr(mod, extension_module_name) - else: - _mod = mod - stub_func = stubs.name_to_func[name] - - if not hasattr(_mod, name): - pytest.skip(f"{mod_name} does not have {name}(), skipping.") - if stub_func is None: - # TODO: Can we make this skip the parameterization entirely? - pytest.skip(f"{name} is not a function, skipping.") - mod_func = getattr(_mod, name) - argspec = inspect.getfullargspec(stub_func) - func_args = argspec.args - if func_args[:1] == ['self']: - func_args = func_args[1:] - nargs = [len(func_args)] - if argspec.defaults: - # The actual default values are checked in the specific tests - nargs.extend([len(func_args) - i for i in range(1, len(argspec.defaults) + 1)]) - - args = [example_argument(arg, name, dtype) for arg in func_args] - if not args: - args = [example_argument('x', name, dtype)] - else: - # Duplicate the last positional argument for the n+1 test. - args = args + [args[-1]] - - kwonlydefaults = argspec.kwonlydefaults or {} - required_kwargs = {arg: example_argument(arg, name, dtype) for arg in argspec.kwonlyargs if arg not in kwonlydefaults} - for n in range(nargs[0]+2): - if name == 'result_type' and n == 0: - # This case is not encoded in the signature, but isn't allowed. - continue - if n in nargs: - doesnt_raise(lambda: mod_func(*args[:n], **required_kwargs)) - elif argspec.varargs: - pass + # We're not interested if the array module has additional arguments, so we + # only iterate through the arguments listed in the spec. + for i, stub_param in enumerate(stub_params): + assert ( + len(params) >= i + 1 + ), f"Argument '{stub_param.name}' missing from signature" + param = params[i] + + # We're not interested in the name if it isn't actually used + if stub_param.kind not in [ + Parameter.POSITIONAL_ONLY, + Parameter.VAR_POSITIONAL, + Parameter.VAR_KEYWORD, + ]: + assert ( + param.name == stub_param.name + ), f"Expected argument '{param.name}' to be named '{stub_param.name}'" + + if ( + stub_param.name in ["x", "x1", "x2"] + and stub_param.kind != Parameter.POSITIONAL_ONLY + ): + pytest.skip( + f"faulty spec - argument {stub_param.name} should be a " + f"{kind_to_str[Parameter.POSITIONAL_ONLY]}" + ) + f_kind = kind_to_str[param.kind] + f_stub_kind = kind_to_str[stub_param.kind] + if stub_param.kind in [ + Parameter.POSITIONAL_OR_KEYWORD, + Parameter.VAR_POSITIONAL, + Parameter.VAR_KEYWORD, + ]: + assert ( + param.kind == stub_param.kind + ), f"{param.name} is a {f_kind}, but should be a {f_stub_kind}" else: - # NumPy ufuncs raise ValueError instead of TypeError - raises((TypeError, ValueError), lambda: mod_func(*args[:n]), f"{name}() should not accept {n} positional arguments") - -@pytest.mark.parametrize('name', params) -def test_function_keyword_only_args(name): - if extension_module(name): - return - - if array_method(name): - _mod = ones((1, 1)) - elif '.' in name: - extension_module_name, name = name.split('.') - _mod = getattr(mod, extension_module_name) - else: - _mod = mod - stub_func = stubs.name_to_func[name] - - if not hasattr(_mod, name): - pytest.skip(f"{mod_name} does not have {name}(), skipping.") - if stub_func is None: - # TODO: Can we make this skip the parameterization entirely? - pytest.skip(f"{name} is not a function, skipping.") - mod_func = getattr(_mod, name) - argspec = inspect.getfullargspec(stub_func) - args = argspec.args - if args[:1] == ['self']: - args = args[1:] - kwonlyargs = argspec.kwonlyargs - kwonlydefaults = argspec.kwonlydefaults or {} - dtype = None - - args = [example_argument(arg, name, dtype) for arg in args] - - for arg in kwonlyargs: - value = example_argument(arg, name, dtype) - # The "only" part of keyword-only is tested by the positional test above. - doesnt_raise(lambda: mod_func(*args, **{arg: value}), - f"{name}() should accept the keyword-only argument {arg!r}") - - # Make sure the default is accepted. These tests are not granular - # enough to test that the default is actually the default, i.e., gives - # the same value if the keyword isn't passed. That is tested in the - # specific function tests. - if arg in kwonlydefaults: - default_value = kwonlydefaults[arg] - doesnt_raise(lambda: mod_func(*args, **{arg: default_value}), - f"{name}() should accept the default value {default_value!r} for the keyword-only argument {arg!r}") + # TODO: allow for kw-only args to be out-of-order + assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD], ( + f"{param.name} is a {f_kind}, " + f"but should be a {f_stub_kind} " + f"(or at least a {kind_to_str[Parameter.POSITIONAL_OR_KEYWORD]})" + ) + + +@pytest.mark.parametrize( + "stub", + [s for stubs in category_to_funcs.values() for s in stubs], + ids=lambda f: f.__name__, +) +def test_func_signature(stub: FunctionType): + assert hasattr(xp, stub.__name__), f"{stub.__name__} not found in array module" + func = getattr(xp, stub.__name__) + _test_signature(func, stub) + + +extension_and_stub_params = [] +for ext, stubs in extension_to_funcs.items(): + for stub in stubs: + p = pytest.param( + ext, stub, id=f"{ext}.{stub.__name__}", marks=pytest.mark.xp_extension(ext) + ) + extension_and_stub_params.append(p) + + +@pytest.mark.parametrize("extension, stub", extension_and_stub_params) +def test_extension_func_signature(extension: str, stub: FunctionType): + mod = getattr(xp, extension) + assert hasattr( + mod, stub.__name__ + ), f"{stub.__name__} not found in {extension} extension" + func = getattr(mod, stub.__name__) + _test_signature(func, stub) + + +@pytest.mark.parametrize("stub", array_methods, ids=lambda f: f.__name__) +@given(x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes())) +def test_array_method_signature(stub: FunctionType, x): + assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" + method = getattr(x, stub.__name__) + # Ignore 'self' arg in stub, which won't be present in instantiated objects. + _test_signature(method, stub, ignore_first_stub_param=True) From 8f3bfdc2d77f0ad0573453b2f59f54f6b1bea504 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 5 Apr 2022 09:52:09 +0100 Subject: [PATCH 02/16] Create `dh.func_in_dtypes` from parsing the spec --- array_api_tests/dtype_helpers.py | 86 ++++++++++---------------------- 1 file changed, 25 insertions(+), 61 deletions(-) diff --git a/array_api_tests/dtype_helpers.py b/array_api_tests/dtype_helpers.py index c8e76a90..42edec0c 100644 --- a/array_api_tests/dtype_helpers.py +++ b/array_api_tests/dtype_helpers.py @@ -1,10 +1,13 @@ +import re from collections.abc import Mapping from functools import lru_cache -from typing import Any, NamedTuple, Sequence, Tuple, Union +from inspect import signature +from typing import Any, Dict, NamedTuple, Sequence, Tuple, Union from warnings import warn from . import _array_module as xp from ._array_module import _UndefinedStub +from .stubs import name_to_func from .typing import DataType, ScalarType __all__ = [ @@ -242,67 +245,28 @@ def result_type(*dtypes: DataType): return result -func_in_dtypes = { - # elementwise - "abs": numeric_dtypes, - "acos": float_dtypes, - "acosh": float_dtypes, - "add": numeric_dtypes, - "asin": float_dtypes, - "asinh": float_dtypes, - "atan": float_dtypes, - "atan2": float_dtypes, - "atanh": float_dtypes, - "bitwise_and": bool_and_all_int_dtypes, - "bitwise_invert": bool_and_all_int_dtypes, - "bitwise_left_shift": all_int_dtypes, - "bitwise_or": bool_and_all_int_dtypes, - "bitwise_right_shift": all_int_dtypes, - "bitwise_xor": bool_and_all_int_dtypes, - "ceil": numeric_dtypes, - "cos": float_dtypes, - "cosh": float_dtypes, - "divide": float_dtypes, - "equal": all_dtypes, - "exp": float_dtypes, - "expm1": float_dtypes, - "floor": numeric_dtypes, - "floor_divide": numeric_dtypes, - "greater": numeric_dtypes, - "greater_equal": numeric_dtypes, - "isfinite": numeric_dtypes, - "isinf": numeric_dtypes, - "isnan": numeric_dtypes, - "less": numeric_dtypes, - "less_equal": numeric_dtypes, - "log": float_dtypes, - "logaddexp": float_dtypes, - "log10": float_dtypes, - "log1p": float_dtypes, - "log2": float_dtypes, - "logical_and": (xp.bool,), - "logical_not": (xp.bool,), - "logical_or": (xp.bool,), - "logical_xor": (xp.bool,), - "multiply": numeric_dtypes, - "negative": numeric_dtypes, - "not_equal": all_dtypes, - "positive": numeric_dtypes, - "pow": numeric_dtypes, - "remainder": numeric_dtypes, - "round": numeric_dtypes, - "sign": numeric_dtypes, - "sin": float_dtypes, - "sinh": float_dtypes, - "sqrt": float_dtypes, - "square": numeric_dtypes, - "subtract": numeric_dtypes, - "tan": float_dtypes, - "tanh": float_dtypes, - "trunc": numeric_dtypes, - # searching - "where": all_dtypes, +r_in_dtypes = re.compile("x1?: array\n.+Should have an? (.+) data type.") +r_int_note = re.compile( + "If one or both of the input arrays have integer data types, " + "the result is implementation-dependent" +) +category_to_dtypes = { + "boolean": (xp.bool,), + "integer": all_int_dtypes, + "floating-point": float_dtypes, + "numeric": numeric_dtypes, + "integer or boolean": bool_and_all_int_dtypes, } +func_in_dtypes: Dict[str, Tuple[DataType, ...]] = {} +for name, func in name_to_func.items(): + if m := r_in_dtypes.search(func.__doc__): + dtype_category = m.group(1) + if dtype_category == "numeric" and r_int_note.search(func.__doc__): + dtype_category = "floating-point" + dtypes = category_to_dtypes[dtype_category] + func_in_dtypes[name] = dtypes + elif any("x" in name for name in signature(func).parameters.keys()): + func_in_dtypes[name] = all_dtypes func_returns_bool = { From 6071f44f1e21b923780e4bece9c48fbb5e3ea636 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 5 Apr 2022 09:57:32 +0100 Subject: [PATCH 03/16] Special case `expm1` input dtypes for now --- array_api_tests/dtype_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/array_api_tests/dtype_helpers.py b/array_api_tests/dtype_helpers.py index 42edec0c..d4ebce26 100644 --- a/array_api_tests/dtype_helpers.py +++ b/array_api_tests/dtype_helpers.py @@ -267,6 +267,8 @@ def result_type(*dtypes: DataType): func_in_dtypes[name] = dtypes elif any("x" in name for name in signature(func).parameters.keys()): func_in_dtypes[name] = all_dtypes +# See https://github.com/data-apis/array-api/pull/413 +func_in_dtypes["expm1"] = float_dtypes func_returns_bool = { From ce2e2c0f11153952852ffc7465c713d43a0fd366 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 5 Apr 2022 10:45:04 +0100 Subject: [PATCH 04/16] Hard-code array scalar casting input dtypes for `dh.func_in_dtypes` --- array_api_tests/dtype_helpers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/array_api_tests/dtype_helpers.py b/array_api_tests/dtype_helpers.py index d4ebce26..40242769 100644 --- a/array_api_tests/dtype_helpers.py +++ b/array_api_tests/dtype_helpers.py @@ -406,6 +406,13 @@ def result_type(*dtypes: DataType): func_returns_bool[iop] = func_returns_bool[op] +func_in_dtypes["__bool__"] = (xp.bool,) +func_in_dtypes["__int__"] = all_int_dtypes +func_in_dtypes["__index__"] = all_int_dtypes +func_in_dtypes["__float__"] = float_dtypes +func_in_dtypes["__dlpack__"] = numeric_dtypes + + @lru_cache def fmt_types(types: Tuple[Union[DataType, ScalarType], ...]) -> str: f_types = [] From 828627b3a29e7e84b803b177e4ed6f5e99f9ec26 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 5 Apr 2022 12:57:02 +0100 Subject: [PATCH 05/16] Skip alias stubs and fallback on source stubs, e.g. for `matmul` --- array_api_tests/stubs.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/array_api_tests/stubs.py b/array_api_tests/stubs.py index 1ff1e1b6..35cc885f 100644 --- a/array_api_tests/stubs.py +++ b/array_api_tests/stubs.py @@ -40,18 +40,29 @@ if name.endswith("_functions"): category = name.replace("_functions", "") objects = [getattr(mod, name) for name in mod.__all__] - assert all(isinstance(o, FunctionType) for o in objects) + assert all(isinstance(o, FunctionType) for o in objects) # sanity check category_to_funcs[category] = objects +all_funcs = [] +for funcs in [array_methods, *category_to_funcs.values()]: + all_funcs.extend(funcs) +name_to_func: Dict[str, FunctionType] = {f.__name__: f for f in all_funcs} + EXTENSIONS: str = ["linalg"] extension_to_funcs: Dict[str, List[FunctionType]] = {} for ext in EXTENSIONS: mod = name_to_mod[ext] objects = [getattr(mod, name) for name in mod.__all__] - assert all(isinstance(o, FunctionType) for o in objects) - extension_to_funcs[ext] = objects + assert all(isinstance(o, FunctionType) for o in objects) # sanity check + funcs = [] + for func in objects: + if "Alias" in func.__doc__: + funcs.append(name_to_func[func.__name__]) + else: + funcs.append(func) + extension_to_funcs[ext] = funcs -all_funcs = [] -for funcs in [array_methods, *category_to_funcs.values(), *extension_to_funcs.values()]: - all_funcs.extend(funcs) -name_to_func: Dict[str, FunctionType] = {f.__name__: f for f in all_funcs} +for funcs in extension_to_funcs.values(): + for func in funcs: + if func.__name__ not in name_to_func.keys(): + name_to_func[func.__name__] = func From 6206e2fe5453b645cbfd3384b7a13b73ed60216e Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 5 Apr 2022 18:07:04 +0100 Subject: [PATCH 06/16] Rudimentary support for uninspectable signatures --- array_api_tests/dtype_helpers.py | 8 +- array_api_tests/test_signatures.py | 232 ++++++++++++++++++++--------- 2 files changed, 166 insertions(+), 74 deletions(-) diff --git a/array_api_tests/dtype_helpers.py b/array_api_tests/dtype_helpers.py index 40242769..94576ee3 100644 --- a/array_api_tests/dtype_helpers.py +++ b/array_api_tests/dtype_helpers.py @@ -245,7 +245,8 @@ def result_type(*dtypes: DataType): return result -r_in_dtypes = re.compile("x1?: array\n.+Should have an? (.+) data type.") +r_alias = re.compile("[aA]lias") +r_in_dtypes = re.compile("x1?: array\n.+have an? (.+) data type.") r_int_note = re.compile( "If one or both of the input arrays have integer data types, " "the result is implementation-dependent" @@ -331,6 +332,8 @@ def result_type(*dtypes: DataType): "trunc": False, # searching "where": False, + # linalg + "matmul": False, } @@ -374,7 +377,7 @@ def result_type(*dtypes: DataType): "__gt__": "greater", "__le__": "less_equal", "__lt__": "less", - # '__matmul__': 'matmul', # TODO: support matmul + "__matmul__": "matmul", "__mod__": "remainder", "__mul__": "multiply", "__ne__": "not_equal", @@ -410,6 +413,7 @@ def result_type(*dtypes: DataType): func_in_dtypes["__int__"] = all_int_dtypes func_in_dtypes["__index__"] = all_int_dtypes func_in_dtypes["__float__"] = float_dtypes +func_in_dtypes["from_dlpack"] = numeric_dtypes func_in_dtypes["__dlpack__"] = numeric_dtypes diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index a3253bb7..418574f5 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -1,17 +1,44 @@ +""" +We're not interested in being 100% strict - instead we focus on areas which +could affect interop, e.g. with + + def add(x1, x2, /): + ... + +x1 and x2 don't need to be pos-only for the purposes of interoperability, but with + + def squeeze(x, /, axis): + ... + +axis has to be pos-or-keyword to support both styles + + >>> squeeze(x, 0) + ... + >>> squeeze(x, axis=0) + ... + +""" +from collections import defaultdict from inspect import Parameter, Signature, signature +from itertools import chain from types import FunctionType -from typing import Callable, Dict +from typing import Callable, DefaultDict, Dict, List import pytest from hypothesis import given +from hypothesis import strategies as st +from . import dtype_helpers as dh from . import hypothesis_helpers as hh from . import xps +from ._array_module import _UndefinedStub from ._array_module import mod as xp from .stubs import array_methods, category_to_funcs, extension_to_funcs +from .typing import DataType, Shape pytestmark = pytest.mark.ci + kind_to_str: Dict[Parameter, str] = { Parameter.POSITIONAL_OR_KEYWORD: "normal argument", Parameter.POSITIONAL_ONLY: "pos-only argument", @@ -20,91 +47,149 @@ Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument", } +VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD) -def _test_signature( - func: Callable, stub: FunctionType, ignore_first_stub_param: bool = False -): - """ - Signature of function is correct enough to not affect interoperability - - We're not interested in being 100% strict - instead we focus on areas which - could affect interop, e.g. with - - def add(x1, x2, /): - ... - x1 and x2 don't need to be pos-only for the purposes of interoperability, but with - - def squeeze(x, /, axis): - ... - - axis has to be pos-or-keyword to support both styles - - >>> squeeze(x, 0) - ... - >>> squeeze(x, axis=0) - ... - - """ - try: - sig = signature(func) - except ValueError: - pytest.skip( - msg=f"type({stub.__name__})={type(func)} not supported by inspect.signature()" - ) +def _test_inspectable_func(sig: Signature, stub_sig: Signature): params = list(sig.parameters.values()) - - stub_sig = signature(stub) stub_params = list(stub_sig.parameters.values()) - if ignore_first_stub_param: - stub_params = stub_params[1:] - stub = Signature( - parameters=stub_params, return_annotation=stub_sig.return_annotation - ) - # We're not interested if the array module has additional arguments, so we # only iterate through the arguments listed in the spec. for i, stub_param in enumerate(stub_params): - assert ( - len(params) >= i + 1 - ), f"Argument '{stub_param.name}' missing from signature" - param = params[i] + if sig is not None: + assert ( + len(params) >= i + 1 + ), f"Argument '{stub_param.name}' missing from signature" + param = params[i] # We're not interested in the name if it isn't actually used - if stub_param.kind not in [ + if sig is not None and stub_param.kind not in [ Parameter.POSITIONAL_ONLY, - Parameter.VAR_POSITIONAL, - Parameter.VAR_KEYWORD, + *VAR_KINDS, ]: assert ( param.name == stub_param.name ), f"Expected argument '{param.name}' to be named '{stub_param.name}'" - if ( - stub_param.name in ["x", "x1", "x2"] - and stub_param.kind != Parameter.POSITIONAL_ONLY - ): - pytest.skip( - f"faulty spec - argument {stub_param.name} should be a " - f"{kind_to_str[Parameter.POSITIONAL_ONLY]}" - ) - f_kind = kind_to_str[param.kind] f_stub_kind = kind_to_str[stub_param.kind] - if stub_param.kind in [ - Parameter.POSITIONAL_OR_KEYWORD, - Parameter.VAR_POSITIONAL, - Parameter.VAR_KEYWORD, - ]: - assert ( - param.kind == stub_param.kind - ), f"{param.name} is a {f_kind}, but should be a {f_stub_kind}" + if stub_param.kind in [Parameter.POSITIONAL_OR_KEYWORD, *VAR_KINDS]: + if sig is not None: + assert param.kind == stub_param.kind, ( + f"{param.name} is a {kind_to_str[param.kind]}, " + f"but should be a {f_stub_kind}" + ) + else: + pass else: # TODO: allow for kw-only args to be out-of-order - assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD], ( - f"{param.name} is a {f_kind}, " - f"but should be a {f_stub_kind} " - f"(or at least a {kind_to_str[Parameter.POSITIONAL_OR_KEYWORD]})" + if sig is not None: + assert param.kind in [ + stub_param.kind, + Parameter.POSITIONAL_OR_KEYWORD, + ], ( + f"{param.name} is a {kind_to_str[param.kind]}, " + f"but should be a {f_stub_kind} " + f"(or at least a {kind_to_str[Parameter.POSITIONAL_OR_KEYWORD]})" + ) + else: + pass + +def shapes(**kw) -> st.SearchStrategy[Shape]: + if "min_side" not in kw.keys(): + kw["min_side"] = 1 + return hh.shapes(**kw) + + +matrixy_funcs: List[str] = [ + f.__name__ + for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"]) +] +matrixy_funcs += ["__matmul__", "triu", "tril"] +func_to_shapes: DefaultDict[str, st.SearchStrategy[Shape]] = defaultdict( + shapes, + { + **{k: st.just(()) for k in ["__bool__", "__int__", "__index__", "__float__"]}, + "sort": shapes(min_dims=1), # for axis=-1, + **{k: shapes(min_dims=2) for k in matrixy_funcs}, + # Override for some matrixy functions + "cross": shapes(min_side=3, max_side=3, min_dims=3, max_dims=3), + "outer": shapes(min_dims=1, max_dims=1), + }, +) + + +def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: + if func_name in dh.func_in_dtypes.keys(): + dtypes = dh.func_in_dtypes[func_name] + if hh.FILTER_UNDEFINED_DTYPES: + dtypes = [d for d in dtypes if not isinstance(d, _UndefinedStub)] + return st.sampled_from(dtypes) + else: + return xps.scalar_dtypes() + + +@given(data=st.data()) +def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature, data): + if func_name in ["cholesky", "inv"]: + func(xp.asarray([[1.0, 0.0], [0.0, 1.0]])) + return + elif func_name == "solve": + func(xp.asarray([[1.0, 2.0], [3.0, 5.0]]), xp.asarray([1.0, 2.0])) + return + + pos_argname_to_example_value = {} + normal_argname_to_example_value = {} + kw_argname_to_example_value = {} + for stub_param in stub_sig.parameters.values(): + if stub_param.name in ["x", "x1"]: + dtypes = get_dtypes_strategy(func_name) + shapes = func_to_shapes[func_name] + example_value = data.draw( + xps.arrays(dtype=dtypes, shape=shapes), label=stub_param.name ) + elif stub_param.name == "x2": + assert "x1" in pos_argname_to_example_value.keys() # sanity check + x1 = pos_argname_to_example_value["x1"] + example_value = data.draw( + xps.arrays(dtype=x1.dtype, shape=x1.shape), label="x2" + ) + else: + if stub_param.default != Parameter.empty: + example_value = stub_param.default + else: + pytest.skip(f"No example value for argument '{stub_param.name}'") + + if stub_param.kind == Parameter.POSITIONAL_ONLY: + pos_argname_to_example_value[stub_param.name] = example_value + elif stub_param.kind == Parameter.POSITIONAL_OR_KEYWORD: + normal_argname_to_example_value[stub_param.name] = example_value + elif stub_param.kind == Parameter.KEYWORD_ONLY: + kw_argname_to_example_value[stub_param.name] = example_value + else: + pytest.skip() + + if len(normal_argname_to_example_value) == 0: + func(*pos_argname_to_example_value.values(), **kw_argname_to_example_value) + else: + pass # TODO + + +def _test_func_signature( + func: Callable, stub: FunctionType, ignore_first_stub_param: bool = False +): + stub_sig = signature(stub) + if ignore_first_stub_param: + stub_params = list(stub_sig.parameters.values()) + del stub_params[0] + stub_sig = Signature( + parameters=stub_params, return_annotation=stub_sig.return_annotation + ) + + try: + sig = signature(func) + _test_inspectable_func(sig, stub_sig) + except ValueError: + _test_uninspectable_func(stub.__name__, func, stub_sig) @pytest.mark.parametrize( @@ -115,7 +200,7 @@ def squeeze(x, /, axis): def test_func_signature(stub: FunctionType): assert hasattr(xp, stub.__name__), f"{stub.__name__} not found in array module" func = getattr(xp, stub.__name__) - _test_signature(func, stub) + _test_func_signature(func, stub) extension_and_stub_params = [] @@ -134,13 +219,16 @@ def test_extension_func_signature(extension: str, stub: FunctionType): mod, stub.__name__ ), f"{stub.__name__} not found in {extension} extension" func = getattr(mod, stub.__name__) - _test_signature(func, stub) + _test_func_signature(func, stub) @pytest.mark.parametrize("stub", array_methods, ids=lambda f: f.__name__) -@given(x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes())) -def test_array_method_signature(stub: FunctionType, x): +@given(data=st.data()) +def test_array_method_signature(stub: FunctionType, data): + dtypes = get_dtypes_strategy(stub.__name__) + shapes = func_to_shapes[stub.__name__] + x = data.draw(xps.arrays(dtype=dtypes, shape=shapes), label="x") assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) # Ignore 'self' arg in stub, which won't be present in instantiated objects. - _test_signature(method, stub, ignore_first_stub_param=True) + _test_func_signature(method, stub, ignore_first_stub_param=True) From 5f5d5fd2664c5d1ba16f59c57bada8846875e4ae Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 8 Apr 2022 13:34:55 +0100 Subject: [PATCH 07/16] Test different arg/kwarg arrangements for uninspectable normal args --- array_api_tests/test_signatures.py | 136 ++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 40 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 418574f5..df028be4 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -19,13 +19,14 @@ def squeeze(x, /, axis): """ from collections import defaultdict +from copy import copy from inspect import Parameter, Signature, signature from itertools import chain from types import FunctionType -from typing import Callable, DefaultDict, Dict, List +from typing import Any, Callable, DefaultDict, Dict, List, Literal, Sequence, get_args import pytest -from hypothesis import given +from hypothesis import given, note from hypothesis import strategies as st from . import dtype_helpers as dh @@ -38,8 +39,16 @@ def squeeze(x, /, axis): pytestmark = pytest.mark.ci - -kind_to_str: Dict[Parameter, str] = { +ParameterKind = Literal[ + Parameter.POSITIONAL_ONLY, + Parameter.VAR_POSITIONAL, + Parameter.POSITIONAL_OR_KEYWORD, + Parameter.KEYWORD_ONLY, + Parameter.VAR_KEYWORD, +] +ALL_KINDS = get_args(ParameterKind) +VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD) +kind_to_str: Dict[ParameterKind, str] = { Parameter.POSITIONAL_OR_KEYWORD: "normal argument", Parameter.POSITIONAL_ONLY: "pos-only argument", Parameter.KEYWORD_ONLY: "keyword-only argument", @@ -47,8 +56,6 @@ def squeeze(x, /, axis): Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument", } -VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD) - def _test_inspectable_func(sig: Signature, stub_sig: Signature): params = list(sig.parameters.values()) @@ -89,11 +96,12 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature): ], ( f"{param.name} is a {kind_to_str[param.kind]}, " f"but should be a {f_stub_kind} " - f"(or at least a {kind_to_str[Parameter.POSITIONAL_OR_KEYWORD]})" + f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})" ) else: pass + def shapes(**kw) -> st.SearchStrategy[Shape]: if "min_side" not in kw.keys(): kw["min_side"] = 1 @@ -111,7 +119,7 @@ def shapes(**kw) -> st.SearchStrategy[Shape]: **{k: st.just(()) for k in ["__bool__", "__int__", "__index__", "__float__"]}, "sort": shapes(min_dims=1), # for axis=-1, **{k: shapes(min_dims=2) for k in matrixy_funcs}, - # Override for some matrixy functions + # Overwrite min_dims=2 shapes for some matrixy functions "cross": shapes(min_side=3, max_side=3, min_dims=3, max_dims=3), "outer": shapes(min_dims=1, max_dims=1), }, @@ -128,50 +136,98 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: return xps.scalar_dtypes() +func_to_example_values: Dict[str, Dict[ParameterKind, Dict[str, Any]]] = { + "broadcast_to": { + Parameter.POSITIONAL_ONLY: {"x": xp.asarray([0, 1])}, + Parameter.POSITIONAL_OR_KEYWORD: {"shape": (1, 2)}, + }, + "cholesky": { + Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])} + }, + "inv": {Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}}, +} + + +def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]): + f_sig = f"{func_name}(" + f_sig += ", ".join(str(a) for a in args) + if len(kwargs) != 0: + if len(args) != 0: + f_sig += ", " + f_sig += ", ".join(f"{k}={v}" for k, v in kwargs.items()) + f_sig += ")" + return f_sig + + @given(data=st.data()) def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature, data): - if func_name in ["cholesky", "inv"]: - func(xp.asarray([[1.0, 0.0], [0.0, 1.0]])) - return - elif func_name == "solve": - func(xp.asarray([[1.0, 2.0], [3.0, 5.0]]), xp.asarray([1.0, 2.0])) - return - - pos_argname_to_example_value = {} - normal_argname_to_example_value = {} - kw_argname_to_example_value = {} - for stub_param in stub_sig.parameters.values(): - if stub_param.name in ["x", "x1"]: + example_values: Dict[ParameterKind, Dict[str, Any]] = func_to_example_values.get( + func_name, {} + ) + for kind in ALL_KINDS: + example_values.setdefault(kind, {}) + + for param in stub_sig.parameters.values(): + for name_to_value in example_values.values(): + if param.name in name_to_value.keys(): + continue + + if param.default != Parameter.empty: + example_value = param.default + elif param.name in ["x", "x1"]: dtypes = get_dtypes_strategy(func_name) shapes = func_to_shapes[func_name] example_value = data.draw( - xps.arrays(dtype=dtypes, shape=shapes), label=stub_param.name + xps.arrays(dtype=dtypes, shape=shapes), label=param.name ) - elif stub_param.name == "x2": - assert "x1" in pos_argname_to_example_value.keys() # sanity check - x1 = pos_argname_to_example_value["x1"] + elif param.name == "x2": + # sanity check + assert "x1" in example_values[Parameter.POSITIONAL_ONLY].keys() + x1 = example_values[Parameter.POSITIONAL_ONLY]["x1"] example_value = data.draw( xps.arrays(dtype=x1.dtype, shape=x1.shape), label="x2" ) + elif param.name == "axes": + example_value = () + elif param.name == "shape": + example_value = () else: - if stub_param.default != Parameter.empty: - example_value = stub_param.default - else: - pytest.skip(f"No example value for argument '{stub_param.name}'") - - if stub_param.kind == Parameter.POSITIONAL_ONLY: - pos_argname_to_example_value[stub_param.name] = example_value - elif stub_param.kind == Parameter.POSITIONAL_OR_KEYWORD: - normal_argname_to_example_value[stub_param.name] = example_value - elif stub_param.kind == Parameter.KEYWORD_ONLY: - kw_argname_to_example_value[stub_param.name] = example_value - else: - pytest.skip() + pytest.skip(f"No example value for argument '{param.name}'") - if len(normal_argname_to_example_value) == 0: - func(*pos_argname_to_example_value.values(), **kw_argname_to_example_value) + if param.kind in VAR_KINDS: + pytest.skip("TODO") + example_values[param.kind][param.name] = example_value + + if len(example_values[Parameter.POSITIONAL_OR_KEYWORD]) == 0: + f_func = make_pretty_func( + func_name, + example_values[Parameter.POSITIONAL_ONLY].values(), + example_values[Parameter.KEYWORD_ONLY], + ) + note(f"trying {f_func}") + func( + *example_values[Parameter.POSITIONAL_ONLY].values(), + **example_values[Parameter.KEYWORD_ONLY], + ) else: - pass # TODO + either_argname_value_pairs = list( + example_values[Parameter.POSITIONAL_OR_KEYWORD].items() + ) + n_either_args = len(either_argname_value_pairs) + for n_extra_args in reversed(range(n_either_args + 1)): + extra_args = [v for _, v in either_argname_value_pairs[:n_extra_args]] + if n_extra_args < n_either_args: + extra_kwargs = dict(either_argname_value_pairs[n_extra_args:]) + else: + extra_kwargs = {} + args = list(example_values[Parameter.POSITIONAL_ONLY].values()) + args += extra_args + kwargs = copy(example_values[Parameter.KEYWORD_ONLY]) + if len(extra_kwargs) != 0: + kwargs.update(extra_kwargs) + f_func = make_pretty_func(func_name, args, kwargs) + note(f"trying {f_func}") + func(*args, **kwargs) def _test_func_signature( From d6a56b4c2dc19fb83efee190190d83026fd44f25 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 11:48:45 +0100 Subject: [PATCH 08/16] Streamline uninspectable testing, just skipping awkward cases --- array_api_tests/test_signatures.py | 204 ++++++++++++----------------- 1 file changed, 81 insertions(+), 123 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index df028be4..3c8d9f51 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -18,15 +18,13 @@ def squeeze(x, /, axis): ... """ -from collections import defaultdict from copy import copy from inspect import Parameter, Signature, signature from itertools import chain from types import FunctionType -from typing import Any, Callable, DefaultDict, Dict, List, Literal, Sequence, get_args +from typing import Any, Callable, Dict, List, Literal, Sequence, get_args import pytest -from hypothesis import given, note from hypothesis import strategies as st from . import dtype_helpers as dh @@ -35,7 +33,7 @@ def squeeze(x, /, axis): from ._array_module import _UndefinedStub from ._array_module import mod as xp from .stubs import array_methods, category_to_funcs, extension_to_funcs -from .typing import DataType, Shape +from .typing import DataType pytestmark = pytest.mark.ci @@ -53,7 +51,7 @@ def squeeze(x, /, axis): Parameter.POSITIONAL_ONLY: "pos-only argument", Parameter.KEYWORD_ONLY: "keyword-only argument", Parameter.VAR_POSITIONAL: "star-args (i.e. *args) argument", - Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument", + Parameter.VAR_KEYWORD: "star-kwonly (i.e. **kwonly) argument", } @@ -63,14 +61,13 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature): # We're not interested if the array module has additional arguments, so we # only iterate through the arguments listed in the spec. for i, stub_param in enumerate(stub_params): - if sig is not None: - assert ( - len(params) >= i + 1 - ), f"Argument '{stub_param.name}' missing from signature" - param = params[i] + assert ( + len(params) >= i + 1 + ), f"Argument '{stub_param.name}' missing from signature" + param = params[i] # We're not interested in the name if it isn't actually used - if sig is not None and stub_param.kind not in [ + if stub_param.kind not in [ Parameter.POSITIONAL_ONLY, *VAR_KINDS, ]: @@ -80,50 +77,17 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature): f_stub_kind = kind_to_str[stub_param.kind] if stub_param.kind in [Parameter.POSITIONAL_OR_KEYWORD, *VAR_KINDS]: - if sig is not None: - assert param.kind == stub_param.kind, ( - f"{param.name} is a {kind_to_str[param.kind]}, " - f"but should be a {f_stub_kind}" - ) - else: - pass + assert param.kind == stub_param.kind, ( + f"{param.name} is a {kind_to_str[param.kind]}, " + f"but should be a {f_stub_kind}" + ) else: # TODO: allow for kw-only args to be out-of-order - if sig is not None: - assert param.kind in [ - stub_param.kind, - Parameter.POSITIONAL_OR_KEYWORD, - ], ( - f"{param.name} is a {kind_to_str[param.kind]}, " - f"but should be a {f_stub_kind} " - f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})" - ) - else: - pass - - -def shapes(**kw) -> st.SearchStrategy[Shape]: - if "min_side" not in kw.keys(): - kw["min_side"] = 1 - return hh.shapes(**kw) - - -matrixy_funcs: List[str] = [ - f.__name__ - for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"]) -] -matrixy_funcs += ["__matmul__", "triu", "tril"] -func_to_shapes: DefaultDict[str, st.SearchStrategy[Shape]] = defaultdict( - shapes, - { - **{k: st.just(()) for k in ["__bool__", "__int__", "__index__", "__float__"]}, - "sort": shapes(min_dims=1), # for axis=-1, - **{k: shapes(min_dims=2) for k in matrixy_funcs}, - # Overwrite min_dims=2 shapes for some matrixy functions - "cross": shapes(min_side=3, max_side=3, min_dims=3, max_dims=3), - "outer": shapes(min_dims=1, max_dims=1), - }, -) + assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD,], ( + f"{param.name} is a {kind_to_str[param.kind]}, " + f"but should be a {f_stub_kind} " + f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})" + ) def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: @@ -136,97 +100,93 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: return xps.scalar_dtypes() -func_to_example_values: Dict[str, Dict[ParameterKind, Dict[str, Any]]] = { - "broadcast_to": { - Parameter.POSITIONAL_ONLY: {"x": xp.asarray([0, 1])}, - Parameter.POSITIONAL_OR_KEYWORD: {"shape": (1, 2)}, - }, - "cholesky": { - Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])} - }, - "inv": {Parameter.POSITIONAL_ONLY: {"x": xp.asarray([[1.0, 0.0], [0.0, 1.0]])}}, -} - - -def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]): +def make_pretty_func(func_name: str, args: Sequence[Any], kwonly: Dict[str, Any]): f_sig = f"{func_name}(" f_sig += ", ".join(str(a) for a in args) - if len(kwargs) != 0: + if len(kwonly) != 0: if len(args) != 0: f_sig += ", " - f_sig += ", ".join(f"{k}={v}" for k, v in kwargs.items()) + f_sig += ", ".join(f"{k}={v}" for k, v in kwonly.items()) f_sig += ")" return f_sig -@given(data=st.data()) -def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature, data): - example_values: Dict[ParameterKind, Dict[str, Any]] = func_to_example_values.get( - func_name, {} - ) - for kind in ALL_KINDS: - example_values.setdefault(kind, {}) +matrixy_funcs: List[str] = [ + f.__name__ + for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"]) +] +matrixy_funcs += ["__matmul__", "triu", "tril"] - for param in stub_sig.parameters.values(): - for name_to_value in example_values.values(): - if param.name in name_to_value.keys(): - continue - if param.default != Parameter.empty: - example_value = param.default +def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature): + skip_msg = ( + f"Signature for {func_name}() is not inspectable " + "and is too troublesome to test for otherwise" + ) + if func_name in [ + "__bool__", + "__int__", + "__index__", + "__float__", + "pow", + "bitwise_left_shift", + "bitwise_right_shift", + "broadcast_to", + "permute_dims", + "sort", + *matrixy_funcs, + ]: + pytest.skip(skip_msg) + + param_to_value: Dict[Parameter, Any] = {} + for param in stub_sig.parameters.values(): + if param.kind in VAR_KINDS: + pytest.skip(skip_msg) + elif param.default != Parameter.empty: + value = param.default elif param.name in ["x", "x1"]: dtypes = get_dtypes_strategy(func_name) - shapes = func_to_shapes[func_name] - example_value = data.draw( - xps.arrays(dtype=dtypes, shape=shapes), label=param.name - ) + value = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example() elif param.name == "x2": # sanity check - assert "x1" in example_values[Parameter.POSITIONAL_ONLY].keys() - x1 = example_values[Parameter.POSITIONAL_ONLY]["x1"] - example_value = data.draw( - xps.arrays(dtype=x1.dtype, shape=x1.shape), label="x2" - ) - elif param.name == "axes": - example_value = () - elif param.name == "shape": - example_value = () + assert "x1" in [p.name for p in param_to_value.keys()] + x1 = next(v for p, v in param_to_value.items() if p.name == "x1") + value = xps.arrays(dtype=x1.dtype, shape=x1.shape).example() else: - pytest.skip(f"No example value for argument '{param.name}'") - - if param.kind in VAR_KINDS: - pytest.skip("TODO") - example_values[param.kind][param.name] = example_value - - if len(example_values[Parameter.POSITIONAL_OR_KEYWORD]) == 0: - f_func = make_pretty_func( - func_name, - example_values[Parameter.POSITIONAL_ONLY].values(), - example_values[Parameter.KEYWORD_ONLY], - ) - note(f"trying {f_func}") - func( - *example_values[Parameter.POSITIONAL_ONLY].values(), - **example_values[Parameter.KEYWORD_ONLY], - ) + pytest.skip(skip_msg) + param_to_value[param] = value + + posonly: List[Any] = [ + v for p, v in param_to_value.items() if p.kind == Parameter.POSITIONAL_ONLY + ] + kwonly: Dict[str, Any] = { + p.name: v for p, v in param_to_value.items() if p.kind == Parameter.KEYWORD_ONLY + } + if ( + sum(p.kind == Parameter.POSITIONAL_OR_KEYWORD for p in param_to_value.keys()) + == 0 + ): + f_func = make_pretty_func(func_name, posonly, kwonly) + print(f"trying {f_func}") + func(*posonly, **kwonly) else: either_argname_value_pairs = list( - example_values[Parameter.POSITIONAL_OR_KEYWORD].items() + (p.name, v) + for p, v in param_to_value.items() + if p.kind == Parameter.POSITIONAL_OR_KEYWORD ) n_either_args = len(either_argname_value_pairs) for n_extra_args in reversed(range(n_either_args + 1)): - extra_args = [v for _, v in either_argname_value_pairs[:n_extra_args]] + extra_posargs = [v for _, v in either_argname_value_pairs[:n_extra_args]] if n_extra_args < n_either_args: extra_kwargs = dict(either_argname_value_pairs[n_extra_args:]) else: extra_kwargs = {} - args = list(example_values[Parameter.POSITIONAL_ONLY].values()) - args += extra_args - kwargs = copy(example_values[Parameter.KEYWORD_ONLY]) - if len(extra_kwargs) != 0: - kwargs.update(extra_kwargs) + args = copy(posonly) + args += extra_posargs + kwargs = {**kwonly, **extra_kwargs} f_func = make_pretty_func(func_name, args, kwargs) - note(f"trying {f_func}") + print(f"trying {f_func}") func(*args, **kwargs) @@ -279,11 +239,9 @@ def test_extension_func_signature(extension: str, stub: FunctionType): @pytest.mark.parametrize("stub", array_methods, ids=lambda f: f.__name__) -@given(data=st.data()) -def test_array_method_signature(stub: FunctionType, data): +def test_array_method_signature(stub: FunctionType): dtypes = get_dtypes_strategy(stub.__name__) - shapes = func_to_shapes[stub.__name__] - x = data.draw(xps.arrays(dtype=dtypes, shape=shapes), label="x") + x = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example() assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) # Ignore 'self' arg in stub, which won't be present in instantiated objects. From e78e3d5f8ad90bcd3d5e04e4d96f7536f2abb9d3 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 12:03:56 +0100 Subject: [PATCH 09/16] Remove support for testing uninspectable normal args --- array_api_tests/test_signatures.py | 56 +++++++++--------------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 3c8d9f51..d7e8134a 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -18,7 +18,6 @@ def squeeze(x, /, axis): ... """ -from copy import copy from inspect import Parameter, Signature, signature from itertools import chain from types import FunctionType @@ -47,11 +46,11 @@ def squeeze(x, /, axis): ALL_KINDS = get_args(ParameterKind) VAR_KINDS = (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD) kind_to_str: Dict[ParameterKind, str] = { - Parameter.POSITIONAL_OR_KEYWORD: "normal argument", + Parameter.POSITIONAL_OR_KEYWORD: "pos or kw argument", Parameter.POSITIONAL_ONLY: "pos-only argument", Parameter.KEYWORD_ONLY: "keyword-only argument", Parameter.VAR_POSITIONAL: "star-args (i.e. *args) argument", - Parameter.VAR_KEYWORD: "star-kwonly (i.e. **kwonly) argument", + Parameter.VAR_KEYWORD: "star-kwargs (i.e. **kwargs) argument", } @@ -100,13 +99,13 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: return xps.scalar_dtypes() -def make_pretty_func(func_name: str, args: Sequence[Any], kwonly: Dict[str, Any]): +def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]): f_sig = f"{func_name}(" f_sig += ", ".join(str(a) for a in args) - if len(kwonly) != 0: + if len(kwargs) != 0: if len(args) != 0: f_sig += ", " - f_sig += ", ".join(f"{k}={v}" for k, v in kwonly.items()) + f_sig += ", ".join(f"{k}={v}" for k, v in kwargs.items()) f_sig += ")" return f_sig @@ -131,8 +130,6 @@ def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature "pow", "bitwise_left_shift", "bitwise_right_shift", - "broadcast_to", - "permute_dims", "sort", *matrixy_funcs, ]: @@ -140,8 +137,10 @@ def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature param_to_value: Dict[Parameter, Any] = {} for param in stub_sig.parameters.values(): - if param.kind in VAR_KINDS: - pytest.skip(skip_msg) + if param.kind in [Parameter.POSITIONAL_OR_KEYWORD, *VAR_KINDS]: + pytest.skip( + skip_msg + f" (because '{param.name}' is a {kind_to_str[param.kind]})" + ) elif param.default != Parameter.empty: value = param.default elif param.name in ["x", "x1"]: @@ -153,41 +152,20 @@ def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature x1 = next(v for p, v in param_to_value.items() if p.name == "x1") value = xps.arrays(dtype=x1.dtype, shape=x1.shape).example() else: - pytest.skip(skip_msg) + pytest.skip( + skip_msg + f" (because no default was found for argument {param.name})" + ) param_to_value[param] = value - posonly: List[Any] = [ + args: List[Any] = [ v for p, v in param_to_value.items() if p.kind == Parameter.POSITIONAL_ONLY ] - kwonly: Dict[str, Any] = { + kwargs: Dict[str, Any] = { p.name: v for p, v in param_to_value.items() if p.kind == Parameter.KEYWORD_ONLY } - if ( - sum(p.kind == Parameter.POSITIONAL_OR_KEYWORD for p in param_to_value.keys()) - == 0 - ): - f_func = make_pretty_func(func_name, posonly, kwonly) - print(f"trying {f_func}") - func(*posonly, **kwonly) - else: - either_argname_value_pairs = list( - (p.name, v) - for p, v in param_to_value.items() - if p.kind == Parameter.POSITIONAL_OR_KEYWORD - ) - n_either_args = len(either_argname_value_pairs) - for n_extra_args in reversed(range(n_either_args + 1)): - extra_posargs = [v for _, v in either_argname_value_pairs[:n_extra_args]] - if n_extra_args < n_either_args: - extra_kwargs = dict(either_argname_value_pairs[n_extra_args:]) - else: - extra_kwargs = {} - args = copy(posonly) - args += extra_posargs - kwargs = {**kwonly, **extra_kwargs} - f_func = make_pretty_func(func_name, args, kwargs) - print(f"trying {f_func}") - func(*args, **kwargs) + f_func = make_pretty_func(func_name, args, kwargs) + print(f"trying {f_func}") + func(*args, **kwargs) def _test_func_signature( From 971cd9542af419dbaed1a50da8da3d25095e4834 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 13:35:34 +0100 Subject: [PATCH 10/16] Use data draws over examples More performative, even for `max_examples=1` --- array_api_tests/test_signatures.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index d7e8134a..273e58e3 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -24,7 +24,9 @@ def squeeze(x, /, axis): from typing import Any, Callable, Dict, List, Literal, Sequence, get_args import pytest +from hypothesis import given, note, settings from hypothesis import strategies as st +from hypothesis.strategies import DataObject from . import dtype_helpers as dh from . import hypothesis_helpers as hh @@ -117,7 +119,11 @@ def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any] matrixy_funcs += ["__matmul__", "triu", "tril"] -def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature): +@given(data=st.data()) +@settings(max_examples=1) +def _test_uninspectable_func( + func_name: str, func: Callable, stub_sig: Signature, data: DataObject +): skip_msg = ( f"Signature for {func_name}() is not inspectable " "and is too troublesome to test for otherwise" @@ -145,12 +151,16 @@ def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature value = param.default elif param.name in ["x", "x1"]: dtypes = get_dtypes_strategy(func_name) - value = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example() + value = data.draw( + xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)), label=param.name + ) elif param.name == "x2": # sanity check assert "x1" in [p.name for p in param_to_value.keys()] x1 = next(v for p, v in param_to_value.items() if p.name == "x1") - value = xps.arrays(dtype=x1.dtype, shape=x1.shape).example() + value = data.draw( + xps.arrays(dtype=x1.dtype, shape=x1.shape), label=param.name + ) else: pytest.skip( skip_msg + f" (because no default was found for argument {param.name})" @@ -164,7 +174,7 @@ def _test_uninspectable_func(func_name: str, func: Callable, stub_sig: Signature p.name: v for p, v in param_to_value.items() if p.kind == Parameter.KEYWORD_ONLY } f_func = make_pretty_func(func_name, args, kwargs) - print(f"trying {f_func}") + note(f"trying {f_func}") func(*args, **kwargs) @@ -217,9 +227,11 @@ def test_extension_func_signature(extension: str, stub: FunctionType): @pytest.mark.parametrize("stub", array_methods, ids=lambda f: f.__name__) -def test_array_method_signature(stub: FunctionType): +@given(st.data()) +@settings(max_examples=1) +def test_array_method_signature(stub: FunctionType, data: DataObject): dtypes = get_dtypes_strategy(stub.__name__) - x = xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)).example() + x = data.draw(xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)), label="x") assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) # Ignore 'self' arg in stub, which won't be present in instantiated objects. From 17a56ba7111232564c0b68f5cddd9ff0e138d1c7 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 13:43:18 +0100 Subject: [PATCH 11/16] Nicer `matrixy_names` definition --- array_api_tests/test_signatures.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 273e58e3..257f3ab5 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -19,7 +19,6 @@ def squeeze(x, /, axis): """ from inspect import Parameter, Signature, signature -from itertools import chain from types import FunctionType from typing import Any, Callable, Dict, List, Literal, Sequence, get_args @@ -112,11 +111,11 @@ def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any] return f_sig -matrixy_funcs: List[str] = [ - f.__name__ - for f in chain(category_to_funcs["linear_algebra"], extension_to_funcs["linalg"]) +matrixy_funcs: List[FunctionType] = [ + *category_to_funcs["linear_algebra"], *extension_to_funcs["linalg"] ] -matrixy_funcs += ["__matmul__", "triu", "tril"] +matrixy_names: List[str] = [f.__name__ for f in matrixy_funcs] +matrixy_names += ["__matmul__", "triu", "tril"] @given(data=st.data()) @@ -137,7 +136,7 @@ def _test_uninspectable_func( "bitwise_left_shift", "bitwise_right_shift", "sort", - *matrixy_funcs, + *matrixy_names, ]: pytest.skip(skip_msg) From 15114b7041a4670071c14e56887b57fa0515083f Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 13:51:15 +0100 Subject: [PATCH 12/16] Test uninspectable array methods --- array_api_tests/test_signatures.py | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 257f3ab5..a522b55c 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -33,7 +33,7 @@ def squeeze(x, /, axis): from ._array_module import _UndefinedStub from ._array_module import mod as xp from .stubs import array_methods, category_to_funcs, extension_to_funcs -from .typing import DataType +from .typing import Array, DataType pytestmark = pytest.mark.ci @@ -112,7 +112,8 @@ def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any] matrixy_funcs: List[FunctionType] = [ - *category_to_funcs["linear_algebra"], *extension_to_funcs["linalg"] + *category_to_funcs["linear_algebra"], + *extension_to_funcs["linalg"], ] matrixy_names: List[str] = [f.__name__ for f in matrixy_funcs] matrixy_names += ["__matmul__", "triu", "tril"] @@ -121,7 +122,7 @@ def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any] @given(data=st.data()) @settings(max_examples=1) def _test_uninspectable_func( - func_name: str, func: Callable, stub_sig: Signature, data: DataObject + func_name: str, func: Callable, stub_sig: Signature, array: Array, data: DataObject ): skip_msg = ( f"Signature for {func_name}() is not inspectable " @@ -153,12 +154,15 @@ def _test_uninspectable_func( value = data.draw( xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)), label=param.name ) - elif param.name == "x2": - # sanity check - assert "x1" in [p.name for p in param_to_value.keys()] - x1 = next(v for p, v in param_to_value.items() if p.name == "x1") + elif param.name in ["x2", "other"]: + if param.name == "x2": + assert "x1" in [p.name for p in param_to_value.keys()] # sanity check + orig = next(v for p, v in param_to_value.items() if p.name == "x1") + else: + assert array is not None # sanity check + orig = array value = data.draw( - xps.arrays(dtype=x1.dtype, shape=x1.shape), label=param.name + xps.arrays(dtype=orig.dtype, shape=orig.shape), label=param.name ) else: pytest.skip( @@ -177,11 +181,11 @@ def _test_uninspectable_func( func(*args, **kwargs) -def _test_func_signature( - func: Callable, stub: FunctionType, ignore_first_stub_param: bool = False -): +def _test_func_signature(func: Callable, stub: FunctionType, array=None): stub_sig = signature(stub) - if ignore_first_stub_param: + # If testing against array, ignore 'self' arg in stub as it won't be present + # in func (which should be an array method). + if array is not None: stub_params = list(stub_sig.parameters.values()) del stub_params[0] stub_sig = Signature( @@ -192,7 +196,7 @@ def _test_func_signature( sig = signature(func) _test_inspectable_func(sig, stub_sig) except ValueError: - _test_uninspectable_func(stub.__name__, func, stub_sig) + _test_uninspectable_func(stub.__name__, func, stub_sig, array) @pytest.mark.parametrize( @@ -233,5 +237,4 @@ def test_array_method_signature(stub: FunctionType, data: DataObject): x = data.draw(xps.arrays(dtype=dtypes, shape=hh.shapes(min_side=1)), label="x") assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) - # Ignore 'self' arg in stub, which won't be present in instantiated objects. - _test_func_signature(method, stub, ignore_first_stub_param=True) + _test_func_signature(method, stub, array=x) From 377c89fdd3922e6f68003b704beec0730237b89d Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 14:08:53 +0100 Subject: [PATCH 13/16] Some docs and syntax niceties --- array_api_tests/test_signatures.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index a522b55c..d162ca55 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -1,4 +1,6 @@ """ +Tests for function/method signatures compliance + We're not interested in being 100% strict - instead we focus on areas which could affect interop, e.g. with @@ -20,7 +22,7 @@ def squeeze(x, /, axis): """ from inspect import Parameter, Signature, signature from types import FunctionType -from typing import Any, Callable, Dict, List, Literal, Sequence, get_args +from typing import Any, Callable, Dict, List, Literal, get_args import pytest from hypothesis import given, note, settings @@ -67,10 +69,7 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature): param = params[i] # We're not interested in the name if it isn't actually used - if stub_param.kind not in [ - Parameter.POSITIONAL_ONLY, - *VAR_KINDS, - ]: + if stub_param.kind not in [Parameter.POSITIONAL_ONLY, *VAR_KINDS]: assert ( param.name == stub_param.name ), f"Expected argument '{param.name}' to be named '{stub_param.name}'" @@ -100,7 +99,7 @@ def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: return xps.scalar_dtypes() -def make_pretty_func(func_name: str, args: Sequence[Any], kwargs: Dict[str, Any]): +def make_pretty_func(func_name: str, *args: Any, **kwargs: Any): f_sig = f"{func_name}(" f_sig += ", ".join(str(a) for a in args) if len(kwargs) != 0: @@ -129,14 +128,18 @@ def _test_uninspectable_func( "and is too troublesome to test for otherwise" ) if func_name in [ + # 0d shapes "__bool__", "__int__", "__index__", "__float__", + # x2 elements must be >=0 "pow", "bitwise_left_shift", "bitwise_right_shift", + # axis default invalid with 0d shapes "sort", + # shape requirements *matrixy_names, ]: pytest.skip(skip_msg) @@ -176,7 +179,7 @@ def _test_uninspectable_func( kwargs: Dict[str, Any] = { p.name: v for p, v in param_to_value.items() if p.kind == Parameter.KEYWORD_ONLY } - f_func = make_pretty_func(func_name, args, kwargs) + f_func = make_pretty_func(func_name, *args, **kwargs) note(f"trying {f_func}") func(*args, **kwargs) @@ -184,7 +187,7 @@ def _test_uninspectable_func( def _test_func_signature(func: Callable, stub: FunctionType, array=None): stub_sig = signature(stub) # If testing against array, ignore 'self' arg in stub as it won't be present - # in func (which should be an array method). + # in func (which should be a method). if array is not None: stub_params = list(stub_sig.parameters.values()) del stub_params[0] From 2f9fb307ff687c2db19561c52fa65ee4a4969597 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Apr 2022 16:21:29 +0100 Subject: [PATCH 14/16] Add failing test for testing out-of-order kw-only args --- array_api_tests/meta/test_signatures.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 array_api_tests/meta/test_signatures.py diff --git a/array_api_tests/meta/test_signatures.py b/array_api_tests/meta/test_signatures.py new file mode 100644 index 00000000..1e34d176 --- /dev/null +++ b/array_api_tests/meta/test_signatures.py @@ -0,0 +1,20 @@ +from inspect import signature + +import pytest + +from ..test_signatures import _test_inspectable_func + + +@pytest.mark.xfail("not implemented") +def test_kwonly(): + def func(*, foo=None, bar=None): + pass + + sig = signature(func) + _test_inspectable_func(sig, sig) + + def reversed_func(*, bar=None, foo=None): + pass + + reversed_sig = signature(reversed_func) + _test_inspectable_func(sig, reversed_sig) From 7e7bd2c07e4f232fd64e7d4692fdb0f9166808e6 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 13 Apr 2022 09:21:41 +0100 Subject: [PATCH 15/16] More meta tests for `test_signatures.py` --- array_api_tests/meta/test_signatures.py | 72 +++++++++++++++++++++---- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/array_api_tests/meta/test_signatures.py b/array_api_tests/meta/test_signatures.py index 1e34d176..f6f40a4c 100644 --- a/array_api_tests/meta/test_signatures.py +++ b/array_api_tests/meta/test_signatures.py @@ -1,20 +1,70 @@ -from inspect import signature +from inspect import Parameter, Signature, signature import pytest from ..test_signatures import _test_inspectable_func -@pytest.mark.xfail("not implemented") -def test_kwonly(): - def func(*, foo=None, bar=None): - pass +def stub(foo, /, bar=None, *, baz=None): + pass - sig = signature(func) - _test_inspectable_func(sig, sig) - def reversed_func(*, bar=None, foo=None): - pass +stub_sig = signature(stub) - reversed_sig = signature(reversed_func) - _test_inspectable_func(sig, reversed_sig) + +@pytest.mark.parametrize( + "sig", + [ + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("baz", Parameter.KEYWORD_ONLY), + ] + ), + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("baz", Parameter.POSITIONAL_OR_KEYWORD), + ] + ), + pytest.param( + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("qux", Parameter.KEYWORD_ONLY), + Parameter("baz", Parameter.KEYWORD_ONLY), + ] + ), + marks=pytest.mark.xfail(reason="out-of-order kwonly args not supported"), + ), + ], +) +def test_good_sig_passes(sig): + _test_inspectable_func(sig, stub_sig) + + +@pytest.mark.parametrize( + "sig", + [ + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.POSITIONAL_ONLY), + Parameter("baz", Parameter.KEYWORD_ONLY), + ] + ), + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.KEYWORD_ONLY), + Parameter("baz", Parameter.KEYWORD_ONLY), + ] + ), + ], +) +def test_raises_on_bad_sig(sig): + with pytest.raises(AssertionError): + _test_inspectable_func(sig, stub_sig) From b172d8414266fdb74398f87cb0e982db4b5552fa Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Thu, 14 Apr 2022 12:55:48 +0100 Subject: [PATCH 16/16] Allow out-of-order kwonly args in signatures --- array_api_tests/meta/test_signatures.py | 17 ++++++--------- array_api_tests/test_signatures.py | 29 +++++++++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/array_api_tests/meta/test_signatures.py b/array_api_tests/meta/test_signatures.py index f6f40a4c..2efe1881 100644 --- a/array_api_tests/meta/test_signatures.py +++ b/array_api_tests/meta/test_signatures.py @@ -29,16 +29,13 @@ def stub(foo, /, bar=None, *, baz=None): Parameter("baz", Parameter.POSITIONAL_OR_KEYWORD), ] ), - pytest.param( - Signature( - [ - Parameter("foo", Parameter.POSITIONAL_ONLY), - Parameter("bar", Parameter.POSITIONAL_OR_KEYWORD), - Parameter("qux", Parameter.KEYWORD_ONLY), - Parameter("baz", Parameter.KEYWORD_ONLY), - ] - ), - marks=pytest.mark.xfail(reason="out-of-order kwonly args not supported"), + Signature( + [ + Parameter("foo", Parameter.POSITIONAL_ONLY), + Parameter("bar", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("qux", Parameter.KEYWORD_ONLY), + Parameter("baz", Parameter.KEYWORD_ONLY), + ] ), ], ) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index d162ca55..2db804b1 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -60,9 +60,15 @@ def squeeze(x, /, axis): def _test_inspectable_func(sig: Signature, stub_sig: Signature): params = list(sig.parameters.values()) stub_params = list(stub_sig.parameters.values()) + + non_kwonly_stub_params = [ + p for p in stub_params if p.kind != Parameter.KEYWORD_ONLY + ] + # sanity check + assert non_kwonly_stub_params == stub_params[: len(non_kwonly_stub_params)] # We're not interested if the array module has additional arguments, so we # only iterate through the arguments listed in the spec. - for i, stub_param in enumerate(stub_params): + for i, stub_param in enumerate(non_kwonly_stub_params): assert ( len(params) >= i + 1 ), f"Argument '{stub_param.name}' missing from signature" @@ -74,19 +80,24 @@ def _test_inspectable_func(sig: Signature, stub_sig: Signature): param.name == stub_param.name ), f"Expected argument '{param.name}' to be named '{stub_param.name}'" - f_stub_kind = kind_to_str[stub_param.kind] if stub_param.kind in [Parameter.POSITIONAL_OR_KEYWORD, *VAR_KINDS]: + f_stub_kind = kind_to_str[stub_param.kind] assert param.kind == stub_param.kind, ( f"{param.name} is a {kind_to_str[param.kind]}, " f"but should be a {f_stub_kind}" ) - else: - # TODO: allow for kw-only args to be out-of-order - assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD,], ( - f"{param.name} is a {kind_to_str[param.kind]}, " - f"but should be a {f_stub_kind} " - f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})" - ) + + kwonly_stub_params = stub_params[len(non_kwonly_stub_params) :] + for stub_param in kwonly_stub_params: + assert ( + stub_param.name in sig.parameters.keys() + ), f"Argument '{stub_param.name}' missing from signature" + param = next(p for p in params if p.name == stub_param.name) + assert param.kind in [stub_param.kind, Parameter.POSITIONAL_OR_KEYWORD,], ( + f"{param.name} is a {kind_to_str[param.kind]}, " + f"but should be a {f_stub_kind} " + f"(or at least a {kind_to_str[ParameterKind.POSITIONAL_OR_KEYWORD]})" + ) def get_dtypes_strategy(func_name: str) -> st.SearchStrategy[DataType]: