Skip to content

Streamline CI workflows with --ci option #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 20, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,7 @@ issues](https://github.com/data-apis/array-api-tests/issues/) to us.
By default, tests for the optional Array API extensions such as
[`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
will be skipped if not present in the specified array module. You can purposely
skip testing extension(s) via the `--disable-extension` option, and likewise
purposely test them via the `--enable-extension` option.
skip testing extension(s) via the `--disable-extension` option.

The tests make heavy use
[Hypothesis](https://hypothesis.readthedocs.io/en/latest/). You can configure
Expand Down
2 changes: 2 additions & 0 deletions array_api_tests/test_array_object.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
from . import xps
from .typing import DataType, Param, Scalar, ScalarType, Shape

pytestmark = pytest.mark.ci


def scalar_objects(dtype: DataType, shape: Shape) -> st.SearchStrategy[List[Scalar]]:
"""Generates scalars or nested sequences which are valid for xp.asarray()"""
Expand Down
2 changes: 2 additions & 0 deletions array_api_tests/test_constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
from ._array_module import mod as xp
from .typing import Array

pytestmark = pytest.mark.ci


def assert_scalar_float(name: str, c: Any):
assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_creation_functions.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import math
import pytest
from itertools import count
from typing import Iterator, NamedTuple, Union

Expand All @@ -14,6 +15,8 @@
from . import xps
from .typing import DataType, Scalar

pytestmark = pytest.mark.ci


class frange(NamedTuple):
start: float
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_linalg.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
from . import _array_module
from ._array_module import linalg

pytestmark = pytest.mark.ci



# Standin strategy for not yet implemented tests
todo = none()
Expand Down
2 changes: 2 additions & 0 deletions array_api_tests/test_manipulation_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
from . import xps
from .typing import Array, Shape

pytestmark = pytest.mark.ci

MAX_SIDE = hh.MAX_ARRAY_SIZE // 64
MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32) # NumPy only supports up to 32 dims

Expand Down
2 changes: 2 additions & 0 deletions array_api_tests/test_operators_and_elementwise_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
from .algos import broadcast_shapes
from .typing import Array, DataType, Param, Scalar

pytestmark = pytest.mark.ci

# When appropiate, this module tests operators alongside their respective
# elementwise methods. We do this by parametrizing a generalised test method
# with every relevant method and operator.
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_searching_functions.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import pytest
from hypothesis import given
from hypothesis import strategies as st

Expand All @@ -9,6 +10,8 @@
from . import xps
from .algos import broadcast_shapes

pytestmark = pytest.mark.ci


@given(
x=xps.arrays(
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_set_functions.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# TODO: disable if opted out, refactor things
import math
import pytest
from collections import Counter, defaultdict

from hypothesis import assume, given
Expand All @@ -11,6 +12,8 @@
from . import shape_helpers as sh
from . import xps

pytestmark = pytest.mark.ci


@given(xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))
def test_unique_all(x):
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_sorting_functions.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import math
import pytest
from typing import Set

from hypothesis import given
Expand All @@ -13,6 +14,8 @@
from . import xps
from .typing import Scalar, Shape

pytestmark = pytest.mark.ci


def assert_scalar_in_set(
func_name: str,
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_statistical_functions.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import math
import pytest
from typing import Optional

from hypothesis import assume, given
Expand All @@ -13,6 +14,8 @@
from . import xps
from .typing import DataType

pytestmark = pytest.mark.ci


def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype]
Expand Down
3 changes: 3 additions & 0 deletions array_api_tests/test_utility_functions.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import pytest
from hypothesis import given
from hypothesis import strategies as st

Expand All @@ -8,6 +9,8 @@
from . import shape_helpers as sh
from . import xps

pytestmark = pytest.mark.ci


@given(
x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)),
Expand Down
95 changes: 48 additions & 47 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -1,67 +1,66 @@
from functools import lru_cache
from pathlib import Path

from pytest import mark
from hypothesis import settings
from pytest import mark

from array_api_tests import _array_module as xp
from array_api_tests._array_module import _UndefinedStub


settings.register_profile('xp_default', deadline=800)
settings.register_profile("xp_default", deadline=800)


def pytest_addoption(parser):
# Hypothesis max examples
# See https://github.com/HypothesisWorks/hypothesis/issues/2434
parser.addoption(
'--hypothesis-max-examples',
'--max-examples',
action='store',
"--hypothesis-max-examples",
"--max-examples",
action="store",
default=None,
help='set the Hypothesis max_examples setting',
help="set the Hypothesis max_examples setting",
)
# Hypothesis deadline
parser.addoption(
'--hypothesis-disable-deadline',
'--disable-deadline',
action='store_true',
help='disable the Hypothesis deadline',
"--hypothesis-disable-deadline",
"--disable-deadline",
action="store_true",
help="disable the Hypothesis deadline",
)
# enable/disable extensions
# disable extensions
parser.addoption(
'--enable-extension',
metavar='ext',
nargs='+',
"--disable-extension",
metavar="ext",
nargs="+",
default=[],
help='enable testing for Array API extension(s)',
help="disable testing for Array API extension(s)",
)
# CI
parser.addoption(
'--disable-extension',
metavar='ext',
nargs='+',
default=[],
help='disable testing for Array API extension(s)',
"--ci",
action="store_true",
help="run just the tests appropiate for CI",
)


def pytest_configure(config):
config.addinivalue_line(
'markers', 'xp_extension(ext): tests an Array API extension'
"markers", "xp_extension(ext): tests an Array API extension"
)
config.addinivalue_line("markers", "primary: primary test")
# Hypothesis
hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
disable_deadline = config.getoption('--hypothesis-disable-deadline')
hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
disable_deadline = config.getoption("--hypothesis-disable-deadline")
profile_settings = {}
if hypothesis_max_examples is not None:
profile_settings['max_examples'] = int(hypothesis_max_examples)
profile_settings["max_examples"] = int(hypothesis_max_examples)
if disable_deadline is not None:
profile_settings['deadline'] = None
profile_settings["deadline"] = None
if profile_settings:
settings.register_profile('xp_override', **profile_settings)
settings.load_profile('xp_override')
settings.register_profile("xp_override", **profile_settings)
settings.load_profile("xp_override")
else:
settings.load_profile('xp_default')
settings.load_profile("xp_default")


@lru_cache
Expand All @@ -73,35 +72,37 @@ def xp_has_ext(ext: str) -> bool:


xfail_ids = []
xfails_path = Path(__file__).parent / 'xfails.txt'
xfails_path = Path(__file__).parent / "xfails.txt"
if xfails_path.exists():
with open(xfails_path) as f:
for line in f:
if line.startswith('array_api_tests'):
id_ = line.strip('\n')
if line.startswith("array_api_tests"):
id_ = line.strip("\n")
xfail_ids.append(id_)


def pytest_collection_modifyitems(config, items):
enabled_exts = config.getoption('--enable-extension')
disabled_exts = config.getoption('--disable-extension')
for ext in enabled_exts:
if ext in disabled_exts:
raise ValueError(f'{ext=} both enabled and disabled')
disabled_exts = config.getoption("--disable-extension")
ci = config.getoption("--ci")
for item in items:
# enable/disable extensions
try:
ext_mark = next(m for m in item.iter_markers() if m.name == 'xp_extension')
markers = list(item.iter_markers())
# skip if disabled or non-existent extension
ext_mark = next((m for m in markers if m.name == "xp_extension"), None)
if ext_mark is not None:
ext = ext_mark.args[0]
if ext in disabled_exts:
item.add_marker(
mark.skip(reason=f'{ext} disabled in --disable-extensions')
mark.skip(reason=f"{ext} disabled in --disable-extensions")
)
elif not ext in enabled_exts and not xp_has_ext(ext):
item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
except StopIteration:
pass
# workflow xfail_ids
elif not xp_has_ext(ext):
item.add_marker(mark.skip(reason=f"{ext} not found in array module"))
# xfail if specified in xfails.txt
for id_ in xfail_ids:
if item.nodeid.startswith(id_):
item.add_marker(mark.xfail(reason='xfails.txt'))
item.add_marker(mark.xfail(reason="xfails.txt"))
break
# skip if test not appropiate for CI
if ci:
ci_mark = next((m for m in markers if m.name == "ci"), None)
if ci_mark is None:
item.add_marker(mark.skip(reason="disabled via --ci"))