Merge pull request #80 from honno/ci-profile

Streamline CI workflows with `--ci` option

Author: honno

README.md

@@ -136,8 +136,7 @@ issues](https://github.com/data-apis/array-api-tests/issues/) to us.
 By default, tests for the optional Array API extensions such as
 [`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
 will be skipped if not present in the specified array module. You can purposely
-skip testing extension(s) via the `--disable-extension` option, and likewise
-purposely test them via the `--enable-extension` option.
+skip testing extension(s) via the `--disable-extension` option.
 
 The tests make heavy use
 [Hypothesis](https://hypothesis.readthedocs.io/en/latest/). You can configure

array_api_tests/test_array_object.py

@@ -14,6 +14,8 @@
 from . import xps
 from .typing import DataType, Param, Scalar, ScalarType, Shape
 
+pytestmark = pytest.mark.ci
+
 
 def scalar_objects(dtype: DataType, shape: Shape) -> st.SearchStrategy[List[Scalar]]:
     """Generates scalars or nested sequences which are valid for xp.asarray()"""

array_api_tests/test_constants.py

@@ -7,6 +7,8 @@
 from ._array_module import mod as xp
 from .typing import Array
 
+pytestmark = pytest.mark.ci
+
 
 def assert_scalar_float(name: str, c: Any):
     assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"

array_api_tests/test_creation_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from itertools import count
 from typing import Iterator, NamedTuple, Union
 
@@ -14,6 +15,8 @@
 from . import xps
 from .typing import DataType, Scalar
 
+pytestmark = pytest.mark.ci
+
 
 class frange(NamedTuple):
     start: float

array_api_tests/test_linalg.py

@@ -35,6 +35,9 @@
 from . import _array_module
 from ._array_module import linalg
 
+pytestmark = pytest.mark.ci
+
+
 
 # Standin strategy for not yet implemented tests
 todo = none()

array_api_tests/test_manipulation_functions.py

@@ -14,6 +14,8 @@
 from . import xps
 from .typing import Array, Shape
 
+pytestmark = pytest.mark.ci
+
 MAX_SIDE = hh.MAX_ARRAY_SIZE // 64
 MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32)  # NumPy only supports up to 32 dims
 

array_api_tests/test_operators_and_elementwise_functions.py

@@ -28,6 +28,8 @@
 from .algos import broadcast_shapes
 from .typing import Array, DataType, Param, Scalar
 
+pytestmark = pytest.mark.ci
+
 # When appropiate, this module tests operators alongside their respective
 # elementwise methods. We do this by parametrizing a generalised test method
 # with every relevant method and operator.

array_api_tests/test_searching_functions.py

@@ -1,3 +1,4 @@
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -9,6 +10,8 @@
 from . import xps
 from .algos import broadcast_shapes
 
+pytestmark = pytest.mark.ci
+
 
 @given(
     x=xps.arrays(

array_api_tests/test_set_functions.py

@@ -1,5 +1,6 @@
 # TODO: disable if opted out, refactor things
 import math
+import pytest
 from collections import Counter, defaultdict
 
 from hypothesis import assume, given
@@ -11,6 +12,8 @@
 from . import shape_helpers as sh
 from . import xps
 
+pytestmark = pytest.mark.ci
+
 
 @given(xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))
 def test_unique_all(x):

array_api_tests/test_sorting_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from typing import Set
 
 from hypothesis import given
@@ -13,6 +14,8 @@
 from . import xps
 from .typing import Scalar, Shape
 
+pytestmark = pytest.mark.ci
+
 
 def assert_scalar_in_set(
     func_name: str,

array_api_tests/test_statistical_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from typing import Optional
 
 from hypothesis import assume, given
@@ -13,6 +14,8 @@
 from . import xps
 from .typing import DataType
 
+pytestmark = pytest.mark.ci
+
 
 def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype]

array_api_tests/test_utility_functions.py

@@ -1,3 +1,4 @@
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -8,6 +9,8 @@
 from . import shape_helpers as sh
 from . import xps
 
+pytestmark = pytest.mark.ci
+
 
 @given(
     x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)),

conftest.py

@@ -1,67 +1,66 @@
 from functools import lru_cache
 from pathlib import Path
 
-from pytest import mark
 from hypothesis import settings
+from pytest import mark
 
 from array_api_tests import _array_module as xp
 from array_api_tests._array_module import _UndefinedStub
 
-
-settings.register_profile('xp_default', deadline=800)
+settings.register_profile("xp_default", deadline=800)
 
 
 def pytest_addoption(parser):
     # Hypothesis max examples
     # See https://github.com/HypothesisWorks/hypothesis/issues/2434
     parser.addoption(
-        '--hypothesis-max-examples',
-        '--max-examples',
-        action='store',
+        "--hypothesis-max-examples",
+        "--max-examples",
+        action="store",
         default=None,
-        help='set the Hypothesis max_examples setting',
+        help="set the Hypothesis max_examples setting",
     )
     # Hypothesis deadline
     parser.addoption(
-        '--hypothesis-disable-deadline',
-        '--disable-deadline',
-        action='store_true',
-        help='disable the Hypothesis deadline',
+        "--hypothesis-disable-deadline",
+        "--disable-deadline",
+        action="store_true",
+        help="disable the Hypothesis deadline",
     )
-    # enable/disable extensions
+    # disable extensions
     parser.addoption(
-        '--enable-extension',
-        metavar='ext',
-        nargs='+',
+        "--disable-extension",
+        metavar="ext",
+        nargs="+",
         default=[],
-        help='enable testing for Array API extension(s)',
+        help="disable testing for Array API extension(s)",
     )
+    # CI
     parser.addoption(
-        '--disable-extension',
-        metavar='ext',
-        nargs='+',
-        default=[],
-        help='disable testing for Array API extension(s)',
+        "--ci",
+        action="store_true",
+        help="run just the tests appropiate for CI",
     )
 
 
 def pytest_configure(config):
     config.addinivalue_line(
-        'markers', 'xp_extension(ext): tests an Array API extension'
+        "markers", "xp_extension(ext): tests an Array API extension"
     )
+    config.addinivalue_line("markers", "primary: primary test")
     # Hypothesis
-    hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
-    disable_deadline = config.getoption('--hypothesis-disable-deadline')
+    hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
+    disable_deadline = config.getoption("--hypothesis-disable-deadline")
     profile_settings = {}
     if hypothesis_max_examples is not None:
-        profile_settings['max_examples'] = int(hypothesis_max_examples)
+        profile_settings["max_examples"] = int(hypothesis_max_examples)
     if disable_deadline is not None:
-        profile_settings['deadline'] = None
+        profile_settings["deadline"] = None
     if profile_settings:
-        settings.register_profile('xp_override', **profile_settings)
-        settings.load_profile('xp_override')
+        settings.register_profile("xp_override", **profile_settings)
+        settings.load_profile("xp_override")
     else:
-        settings.load_profile('xp_default')
+        settings.load_profile("xp_default")
 
 
 @lru_cache
@@ -73,35 +72,37 @@ def xp_has_ext(ext: str) -> bool:
 
 
 xfail_ids = []
-xfails_path = Path(__file__).parent / 'xfails.txt'
+xfails_path = Path(__file__).parent / "xfails.txt"
 if xfails_path.exists():
     with open(xfails_path) as f:
         for line in f:
-            if line.startswith('array_api_tests'):
-                id_ = line.strip('\n')
+            if line.startswith("array_api_tests"):
+                id_ = line.strip("\n")
                 xfail_ids.append(id_)
 
 
 def pytest_collection_modifyitems(config, items):
-    enabled_exts = config.getoption('--enable-extension')
-    disabled_exts = config.getoption('--disable-extension')
-    for ext in enabled_exts:
-        if ext in disabled_exts:
-            raise ValueError(f'{ext=} both enabled and disabled')
+    disabled_exts = config.getoption("--disable-extension")
+    ci = config.getoption("--ci")
     for item in items:
-        # enable/disable extensions
-        try:
-            ext_mark = next(m for m in item.iter_markers() if m.name == 'xp_extension')
+        markers = list(item.iter_markers())
+        # skip if disabled or non-existent extension
+        ext_mark = next((m for m in markers if m.name == "xp_extension"), None)
+        if ext_mark is not None:
             ext = ext_mark.args[0]
             if ext in disabled_exts:
                 item.add_marker(
-                    mark.skip(reason=f'{ext} disabled in --disable-extensions')
+                    mark.skip(reason=f"{ext} disabled in --disable-extensions")
                 )
-            elif not ext in enabled_exts and not xp_has_ext(ext):
-                item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
-        except StopIteration:
-            pass
-        # workflow xfail_ids
+            elif not xp_has_ext(ext):
+                item.add_marker(mark.skip(reason=f"{ext} not found in array module"))
+        # xfail if specified in xfails.txt
         for id_ in xfail_ids:
             if item.nodeid.startswith(id_):
-                item.add_marker(mark.xfail(reason='xfails.txt'))
+                item.add_marker(mark.xfail(reason="xfails.txt"))
+                break
+        # skip if test not appropiate for CI
+        if ci:
+            ci_mark = next((m for m in markers if m.name == "ci"), None)
+            if ci_mark is None:
+                item.add_marker(mark.skip(reason="disabled via --ci"))