diff --git a/README.md b/README.md index 438ad5e9..77cfb4b3 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,7 @@ issues](https://github.com/data-apis/array-api-tests/issues/) to us. By default, tests for the optional Array API extensions such as [`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html) will be skipped if not present in the specified array module. You can purposely -skip testing extension(s) via the `--disable-extension` option, and likewise -purposely test them via the `--enable-extension` option. +skip testing extension(s) via the `--disable-extension` option. The tests make heavy use [Hypothesis](https://hypothesis.readthedocs.io/en/latest/). You can configure diff --git a/array_api_tests/test_array_object.py b/array_api_tests/test_array_object.py index ce6ae596..4b86f96c 100644 --- a/array_api_tests/test_array_object.py +++ b/array_api_tests/test_array_object.py @@ -14,6 +14,8 @@ from . import xps from .typing import DataType, Param, Scalar, ScalarType, Shape +pytestmark = pytest.mark.ci + def scalar_objects(dtype: DataType, shape: Shape) -> st.SearchStrategy[List[Scalar]]: """Generates scalars or nested sequences which are valid for xp.asarray()""" diff --git a/array_api_tests/test_constants.py b/array_api_tests/test_constants.py index cc4eec9f..606bf897 100644 --- a/array_api_tests/test_constants.py +++ b/array_api_tests/test_constants.py @@ -7,6 +7,8 @@ from ._array_module import mod as xp from .typing import Array +pytestmark = pytest.mark.ci + def assert_scalar_float(name: str, c: Any): assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float" diff --git a/array_api_tests/test_creation_functions.py b/array_api_tests/test_creation_functions.py index 4b96151a..8e448339 100644 --- a/array_api_tests/test_creation_functions.py +++ b/array_api_tests/test_creation_functions.py @@ -1,4 +1,5 @@ import math +import pytest from itertools import count from typing import Iterator, NamedTuple, Union @@ -14,6 +15,8 @@ from . import xps from .typing import DataType, Scalar +pytestmark = pytest.mark.ci + class frange(NamedTuple): start: float diff --git a/array_api_tests/test_linalg.py b/array_api_tests/test_linalg.py index 89707d3f..52c8774c 100644 --- a/array_api_tests/test_linalg.py +++ b/array_api_tests/test_linalg.py @@ -35,6 +35,9 @@ from . import _array_module from ._array_module import linalg +pytestmark = pytest.mark.ci + + # Standin strategy for not yet implemented tests todo = none() diff --git a/array_api_tests/test_manipulation_functions.py b/array_api_tests/test_manipulation_functions.py index 12e0c203..1ae28919 100644 --- a/array_api_tests/test_manipulation_functions.py +++ b/array_api_tests/test_manipulation_functions.py @@ -14,6 +14,8 @@ from . import xps from .typing import Array, Shape +pytestmark = pytest.mark.ci + MAX_SIDE = hh.MAX_ARRAY_SIZE // 64 MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32) # NumPy only supports up to 32 dims diff --git a/array_api_tests/test_operators_and_elementwise_functions.py b/array_api_tests/test_operators_and_elementwise_functions.py index 2e14a6c7..f0a21298 100644 --- a/array_api_tests/test_operators_and_elementwise_functions.py +++ b/array_api_tests/test_operators_and_elementwise_functions.py @@ -28,6 +28,8 @@ from .algos import broadcast_shapes from .typing import Array, DataType, Param, Scalar +pytestmark = pytest.mark.ci + # When appropiate, this module tests operators alongside their respective # elementwise methods. We do this by parametrizing a generalised test method # with every relevant method and operator. diff --git a/array_api_tests/test_searching_functions.py b/array_api_tests/test_searching_functions.py index 244e7c24..b6a66086 100644 --- a/array_api_tests/test_searching_functions.py +++ b/array_api_tests/test_searching_functions.py @@ -1,3 +1,4 @@ +import pytest from hypothesis import given from hypothesis import strategies as st @@ -9,6 +10,8 @@ from . import xps from .algos import broadcast_shapes +pytestmark = pytest.mark.ci + @given( x=xps.arrays( diff --git a/array_api_tests/test_set_functions.py b/array_api_tests/test_set_functions.py index e3ceaa0b..9679eaac 100644 --- a/array_api_tests/test_set_functions.py +++ b/array_api_tests/test_set_functions.py @@ -1,5 +1,6 @@ # TODO: disable if opted out, refactor things import math +import pytest from collections import Counter, defaultdict from hypothesis import assume, given @@ -11,6 +12,8 @@ from . import shape_helpers as sh from . import xps +pytestmark = pytest.mark.ci + @given(xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1))) def test_unique_all(x): diff --git a/array_api_tests/test_sorting_functions.py b/array_api_tests/test_sorting_functions.py index cf8f5070..ea375b57 100644 --- a/array_api_tests/test_sorting_functions.py +++ b/array_api_tests/test_sorting_functions.py @@ -1,4 +1,5 @@ import math +import pytest from typing import Set from hypothesis import given @@ -13,6 +14,8 @@ from . import xps from .typing import Scalar, Shape +pytestmark = pytest.mark.ci + def assert_scalar_in_set( func_name: str, diff --git a/array_api_tests/test_statistical_functions.py b/array_api_tests/test_statistical_functions.py index c2fb33db..c955b570 100644 --- a/array_api_tests/test_statistical_functions.py +++ b/array_api_tests/test_statistical_functions.py @@ -1,4 +1,5 @@ import math +import pytest from typing import Optional from hypothesis import assume, given @@ -13,6 +14,8 @@ from . import xps from .typing import DataType +pytestmark = pytest.mark.ci + def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]: dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype] diff --git a/array_api_tests/test_utility_functions.py b/array_api_tests/test_utility_functions.py index c10d0dbd..54052355 100644 --- a/array_api_tests/test_utility_functions.py +++ b/array_api_tests/test_utility_functions.py @@ -1,3 +1,4 @@ +import pytest from hypothesis import given from hypothesis import strategies as st @@ -8,6 +9,8 @@ from . import shape_helpers as sh from . import xps +pytestmark = pytest.mark.ci + @given( x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)), diff --git a/conftest.py b/conftest.py index 64c46df7..8bdb3b83 100644 --- a/conftest.py +++ b/conftest.py @@ -1,67 +1,66 @@ from functools import lru_cache from pathlib import Path -from pytest import mark from hypothesis import settings +from pytest import mark from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub - -settings.register_profile('xp_default', deadline=800) +settings.register_profile("xp_default", deadline=800) def pytest_addoption(parser): # Hypothesis max examples # See https://github.com/HypothesisWorks/hypothesis/issues/2434 parser.addoption( - '--hypothesis-max-examples', - '--max-examples', - action='store', + "--hypothesis-max-examples", + "--max-examples", + action="store", default=None, - help='set the Hypothesis max_examples setting', + help="set the Hypothesis max_examples setting", ) # Hypothesis deadline parser.addoption( - '--hypothesis-disable-deadline', - '--disable-deadline', - action='store_true', - help='disable the Hypothesis deadline', + "--hypothesis-disable-deadline", + "--disable-deadline", + action="store_true", + help="disable the Hypothesis deadline", ) - # enable/disable extensions + # disable extensions parser.addoption( - '--enable-extension', - metavar='ext', - nargs='+', + "--disable-extension", + metavar="ext", + nargs="+", default=[], - help='enable testing for Array API extension(s)', + help="disable testing for Array API extension(s)", ) + # CI parser.addoption( - '--disable-extension', - metavar='ext', - nargs='+', - default=[], - help='disable testing for Array API extension(s)', + "--ci", + action="store_true", + help="run just the tests appropiate for CI", ) def pytest_configure(config): config.addinivalue_line( - 'markers', 'xp_extension(ext): tests an Array API extension' + "markers", "xp_extension(ext): tests an Array API extension" ) + config.addinivalue_line("markers", "primary: primary test") # Hypothesis - hypothesis_max_examples = config.getoption('--hypothesis-max-examples') - disable_deadline = config.getoption('--hypothesis-disable-deadline') + hypothesis_max_examples = config.getoption("--hypothesis-max-examples") + disable_deadline = config.getoption("--hypothesis-disable-deadline") profile_settings = {} if hypothesis_max_examples is not None: - profile_settings['max_examples'] = int(hypothesis_max_examples) + profile_settings["max_examples"] = int(hypothesis_max_examples) if disable_deadline is not None: - profile_settings['deadline'] = None + profile_settings["deadline"] = None if profile_settings: - settings.register_profile('xp_override', **profile_settings) - settings.load_profile('xp_override') + settings.register_profile("xp_override", **profile_settings) + settings.load_profile("xp_override") else: - settings.load_profile('xp_default') + settings.load_profile("xp_default") @lru_cache @@ -73,35 +72,37 @@ def xp_has_ext(ext: str) -> bool: xfail_ids = [] -xfails_path = Path(__file__).parent / 'xfails.txt' +xfails_path = Path(__file__).parent / "xfails.txt" if xfails_path.exists(): with open(xfails_path) as f: for line in f: - if line.startswith('array_api_tests'): - id_ = line.strip('\n') + if line.startswith("array_api_tests"): + id_ = line.strip("\n") xfail_ids.append(id_) def pytest_collection_modifyitems(config, items): - enabled_exts = config.getoption('--enable-extension') - disabled_exts = config.getoption('--disable-extension') - for ext in enabled_exts: - if ext in disabled_exts: - raise ValueError(f'{ext=} both enabled and disabled') + disabled_exts = config.getoption("--disable-extension") + ci = config.getoption("--ci") for item in items: - # enable/disable extensions - try: - ext_mark = next(m for m in item.iter_markers() if m.name == 'xp_extension') + markers = list(item.iter_markers()) + # skip if disabled or non-existent extension + ext_mark = next((m for m in markers if m.name == "xp_extension"), None) + if ext_mark is not None: ext = ext_mark.args[0] if ext in disabled_exts: item.add_marker( - mark.skip(reason=f'{ext} disabled in --disable-extensions') + mark.skip(reason=f"{ext} disabled in --disable-extensions") ) - elif not ext in enabled_exts and not xp_has_ext(ext): - item.add_marker(mark.skip(reason=f'{ext} not found in array module')) - except StopIteration: - pass - # workflow xfail_ids + elif not xp_has_ext(ext): + item.add_marker(mark.skip(reason=f"{ext} not found in array module")) + # xfail if specified in xfails.txt for id_ in xfail_ids: if item.nodeid.startswith(id_): - item.add_marker(mark.xfail(reason='xfails.txt')) + item.add_marker(mark.xfail(reason="xfails.txt")) + break + # skip if test not appropiate for CI + if ci: + ci_mark = next((m for m in markers if m.name == "ci"), None) + if ci_mark is None: + item.add_marker(mark.skip(reason="disabled via --ci"))