Add custom ci mark to skip non-primary tests

Author: honno

array_api_tests/test_array_object.py

@@ -14,6 +14,8 @@
 from . import xps
 from .typing import DataType, Param, Scalar, ScalarType, Shape
 
+pytestmark = pytest.mark.ci
+
 
 def scalar_objects(dtype: DataType, shape: Shape) -> st.SearchStrategy[List[Scalar]]:
     """Generates scalars or nested sequences which are valid for xp.asarray()"""

array_api_tests/test_constants.py

@@ -7,6 +7,8 @@
 from ._array_module import mod as xp
 from .typing import Array
 
+pytestmark = pytest.mark.ci
+
 
 def assert_scalar_float(name: str, c: Any):
     assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"

array_api_tests/test_creation_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from itertools import count
 from typing import Iterator, NamedTuple, Union
 
@@ -14,6 +15,8 @@
 from . import xps
 from .typing import DataType, Scalar
 
+pytestmark = pytest.mark.ci
+
 
 class frange(NamedTuple):
     start: float

array_api_tests/test_linalg.py

@@ -35,6 +35,9 @@
 from . import _array_module
 from ._array_module import linalg
 
+pytestmark = pytest.mark.ci
+
+
 
 # Standin strategy for not yet implemented tests
 todo = none()

array_api_tests/test_manipulation_functions.py

@@ -14,6 +14,8 @@
 from . import xps
 from .typing import Array, Shape
 
+pytestmark = pytest.mark.ci
+
 MAX_SIDE = hh.MAX_ARRAY_SIZE // 64
 MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32)  # NumPy only supports up to 32 dims
 

array_api_tests/test_operators_and_elementwise_functions.py

@@ -28,6 +28,8 @@
 from .algos import broadcast_shapes
 from .typing import Array, DataType, Param, Scalar
 
+pytestmark = pytest.mark.ci
+
 # When appropiate, this module tests operators alongside their respective
 # elementwise methods. We do this by parametrizing a generalised test method
 # with every relevant method and operator.

array_api_tests/test_searching_functions.py

@@ -1,3 +1,4 @@
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -9,6 +10,8 @@
 from . import xps
 from .algos import broadcast_shapes
 
+pytestmark = pytest.mark.ci
+
 
 @given(
     x=xps.arrays(

array_api_tests/test_set_functions.py

@@ -1,5 +1,6 @@
 # TODO: disable if opted out, refactor things
 import math
+import pytest
 from collections import Counter, defaultdict
 
 from hypothesis import assume, given
@@ -11,6 +12,8 @@
 from . import shape_helpers as sh
 from . import xps
 
+pytestmark = pytest.mark.ci
+
 
 @given(xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))
 def test_unique_all(x):

array_api_tests/test_sorting_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from typing import Set
 
 from hypothesis import given
@@ -13,6 +14,8 @@
 from . import xps
 from .typing import Scalar, Shape
 
+pytestmark = pytest.mark.ci
+
 
 def assert_scalar_in_set(
     func_name: str,

array_api_tests/test_statistical_functions.py

@@ -1,4 +1,5 @@
 import math
+import pytest
 from typing import Optional
 
 from hypothesis import assume, given
@@ -13,6 +14,8 @@
 from . import xps
 from .typing import DataType
 
+pytestmark = pytest.mark.ci
+
 
 def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype]

array_api_tests/test_utility_functions.py

@@ -1,3 +1,4 @@
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -8,6 +9,8 @@
 from . import shape_helpers as sh
 from . import xps
 
+pytestmark = pytest.mark.ci
+
 
 @given(
     x=xps.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)),

conftest.py

@@ -35,12 +35,19 @@ def pytest_addoption(parser):
         default=[],
         help="disable testing for Array API extension(s)",
     )
+    # CI
+    parser.addoption(
+        "--ci",
+        action="store_true",
+        help="run just the tests appropiate for CI",
+    )
 
 
 def pytest_configure(config):
     config.addinivalue_line(
         "markers", "xp_extension(ext): tests an Array API extension"
     )
+    config.addinivalue_line("markers", "primary: primary test")
     # Hypothesis
     hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
     disable_deadline = config.getoption("--hypothesis-disable-deadline")
@@ -76,20 +83,26 @@ def xp_has_ext(ext: str) -> bool:
 
 def pytest_collection_modifyitems(config, items):
     disabled_exts = config.getoption("--disable-extension")
+    ci = config.getoption("--ci")
     for item in items:
-        # disable extensions
-        try:
-            ext_mark = next(m for m in item.iter_markers() if m.name == "xp_extension")
+        markers = list(item.iter_markers())
+        # skip if disabled or non-existent extension
+        ext_mark = next((m for m in markers if m.name == "xp_extension"), None)
+        if ext_mark is not None:
             ext = ext_mark.args[0]
             if ext in disabled_exts:
                 item.add_marker(
                     mark.skip(reason=f"{ext} disabled in --disable-extensions")
                 )
             elif not xp_has_ext(ext):
                 item.add_marker(mark.skip(reason=f"{ext} not found in array module"))
-        except StopIteration:
-            pass
-        # workflow xfail_ids
+        # xfail if specified in xfails.txt
         for id_ in xfail_ids:
             if item.nodeid.startswith(id_):
                 item.add_marker(mark.xfail(reason="xfails.txt"))
+                break
+        # skip if test not appropiate for CI
+        if ci:
+            ci_mark = next((m for m in markers if m.name == "ci"), None)
+            if ci_mark is None:
+                item.add_marker(mark.skip(reason="disabled via --ci"))