Merge pull request #29 from honno/skip-extensions

Skip extension tests if the respective extension is not found or is passed via `--disable-extensions`

Author: asmeurer

.github/workflows/numpy.yml

@@ -22,41 +22,26 @@ jobs:
         python -m pip install git+https://github.com/numpy/numpy
         python -m pip install -r requirements.txt
     - name: Run the test suite
+      env:
+        ARRAY_API_TESTS_MODULE: numpy.array_api
       run: |
         # Mark some known issues as XFAIL
-        cat << EOF >> conftest.py
+        cat << EOF >> xfails.txt
+
+        # https://github.com/numpy/numpy/issues/18881
+        array_api_tests/test_creation_functions.py::test_linspace
+        # einsum is not yet completed in the spec
+        array_api_tests/test_signatures.py::test_has_names[einsum]
+        # dlpack support is not yet implemented in NumPy
+        # See https://github.com/numpy/numpy/pull/19083
+        array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]
+        array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]
+        array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]
+        array_api_tests/test_signatures.py::test_function_positional_args[to_device]
+        array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]
+        # floor_divide has an issue related to https://github.com/data-apis/array-api/issues/264
+        array_api_tests/test_elementwise_functions.py::test_floor_divide
 
-        names_to_be_xfailed = (
-            # https://github.com/numpy/numpy/issues/18881
-            "array_api_tests/test_creation_functions.py::test_linspace",
-            # einsum is not yet completed in the spec
-            "array_api_tests/test_signatures.py::test_has_names[einsum]",
-            # The linalg extension is not yet implemented in NumPy
-            "array_api_tests/test_signatures.py::test_has_names[linalg]",
-            # dlpack support is not yet implemented in NumPy. https://github.com/numpy/numpy/pull/19083
-            "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]",
-
-            # Updates to the spec since the last change to numpy.array_api.
-            # These will fail until NumPy is updated.
-            "array_api_tests/test_signatures.py::test_has_names[__index__]",
-            "array_api_tests/test_signatures.py::test_has_names[to_device]",
-            "array_api_tests/test_signatures.py::test_has_names[mT]",
-            "array_api_tests/test_signatures.py::test_has_names[tril]",
-            "array_api_tests/test_signatures.py::test_has_names[triu]",
-            "array_api_tests/test_signatures.py::test_has_names[matrix_transpose]",
-            "array_api_tests/test_signatures.py::test_has_names[permute_dims]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[__index__]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[prod]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[sum]",
-        )
-
-        def pytest_collection_modifyitems(config, items):
-            for item in items:
-                if item.nodeid in names_to_be_xfailed:
-                    item.add_marker("xfail")
         EOF
 
-        ARRAY_API_TESTS_MODULE=numpy.array_api pytest -v -rxXfE
+        pytest -v -rxXfE

README.md

@@ -10,19 +10,23 @@ specification that are not yet tested here.
 
 ## Running the tests
 
+### Setup
+
 To run the tests, first install the testing dependencies
 
-    pip install pytest hypothesis numpy
+    pip install pytest hypothesis
 
 or
 
-    conda install pytest hypothesis numpy
+    conda install pytest hypothesis
+
+as well as the array libraries that you want to test. 
+
+### Specifying the array module
 
-as well as the array libraries that you want to test. (Note, in the future,
-NumPy will be removed as a dependency on the test suite). To run the tests,
-you need to set the array library that is to be tested. There are two ways to
-do this. One way is to set the `ARRAY_API_TESTS_MODULE` environment variable.
-For example
+To run the tests, you need to set the array library that is to be tested. There
+are two ways to do this. One way is to set the `ARRAY_API_TESTS_MODULE`
+environment variable. For example you can set it when running `pytest`
 
     ARRAY_API_TESTS_MODULE=numpy pytest
 
@@ -35,12 +39,21 @@ array_module = None
 
 to
 
-```
+```py
 import numpy as array_module
 ```
 
 (replacing `numpy` with the array module namespace to be tested).
 
+### Specifying test cases
+
+The test suite tries to logically organise its tests so you can find specific
+test cases whilst developing something in particular. So to avoid running the
+rather slow complete suite, you can specify particular test cases like any other
+test suite.
+
+    pytest array_api_tests/test_creation_functions.py::test_zeros
+
 ## Notes on Interpreting Errors
 
 - Some tests cannot be run unless other tests pass first. This is because very
@@ -66,6 +79,12 @@ import numpy as array_module
 
 ## Configuring Tests
 
+By default, tests for the optional Array API extensions such as
+[`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
+will be skipped if not present in the specified array module. You can purposely
+skip testing extension(s) via the `--disable-extension` option, and likewise
+purposely test them via the `--enable-extension` option.
+
 The tests make heavy use of the
 [Hypothesis](https://hypothesis.readthedocs.io/en/latest/) testing library.
 Hypothesis generates random input values for the tests. You can configure how

array_api_tests/meta_tests/__init__.py renamed to array_api_tests/meta/__init__.py

@@ -0,0 +1,9 @@
+from ..test_signatures import extension_module
+
+
+def test_extension_module_is_extension():
+    assert extension_module('linalg')
+
+
+def test_extension_func_is_not_extension():
+    assert not extension_module('linalg.cross')

array_api_tests/meta_tests/test_array_helpers.py renamed to array_api_tests/meta/test_array_helpers.py

@@ -13,6 +13,7 @@
 
 """
 
+import pytest
 from hypothesis import assume, given
 from hypothesis.strategies import (booleans, composite, none, tuples, integers,
                                    shared, sampled_from)
@@ -33,6 +34,7 @@
 from . import _array_module
 from ._array_module import linalg
 
+
 # Standin strategy for not yet implemented tests
 todo = none()
 
@@ -74,6 +76,7 @@ def _test_namedtuple(res, fields, func_name):
         assert hasattr(res, field), f"{func_name}() result namedtuple doesn't have the '{field}' field"
         assert res[i] is getattr(res, field), f"{func_name}() result namedtuple '{field}' field is not in position {i}"
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=positive_definite_matrices(),
     kw=kwargs(upper=booleans())
@@ -121,6 +124,7 @@ def cross_args(draw, dtype_objects=dh.numeric_dtypes):
     )
     return draw(arrays1), draw(arrays2), kw
 
+@pytest.mark.xp_extension('linalg')
 @given(
     cross_args()
 )
@@ -159,6 +163,7 @@ def test_cross(x1_x2_kw):
             ], dtype=res.dtype)
         assert_exactly_equal(res_stack, exact_cross)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=square_matrix_shapes),
 )
@@ -172,6 +177,7 @@ def test_det(x):
 
     # TODO: Test that res actually corresponds to the determinant of x
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=dtypes, shape=matrix_shapes),
     # offset may produce an overflow if it is too large. Supporting offsets
@@ -206,6 +212,7 @@ def true_diag(x_stack):
 
     _test_stacks(linalg.diagonal, x, **kw, res=res, dims=1, true_val=true_diag)
 
+@pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))
 def test_eigh(x):
     res = linalg.eigh(x)
@@ -229,6 +236,7 @@ def test_eigh(x):
     # TODO: Test that res actually corresponds to the eigenvalues and
     # eigenvectors of x
 
+@pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))
 def test_eigvalsh(x):
     res = linalg.eigvalsh(x)
@@ -242,6 +250,7 @@ def test_eigvalsh(x):
 
     # TODO: Test that res actually corresponds to the eigenvalues of x
 
+@pytest.mark.xp_extension('linalg')
 @given(x=invertible_matrices())
 def test_inv(x):
     res = linalg.inv(x)
@@ -286,6 +295,7 @@ def test_matmul(x1, x2):
         assert res.shape == stack_shape + (x1.shape[-2], x2.shape[-1])
         _test_stacks(_array_module.matmul, x1, x2, res=res)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(axis=todo, keepdims=todo, ord=todo)
@@ -295,6 +305,7 @@ def test_matrix_norm(x, kw):
     pass
 
 matrix_power_n = shared(integers(-1000, 1000), key='matrix_power n')
+@pytest.mark.xp_extension('linalg')
 @given(
     # Generate any square matrix if n >= 0 but only invertible matrices if n < 0
     x=matrix_power_n.flatmap(lambda n: invertible_matrices() if n < 0 else
@@ -316,6 +327,7 @@ def test_matrix_power(x, n):
     func = lambda x: linalg.matrix_power(x, n)
     _test_stacks(func, x, res=res, true_val=true_val)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(rtol=todo)
@@ -341,6 +353,7 @@ def test_matrix_transpose(x):
 
     _test_stacks(_array_module.matrix_transpose, x, res=res, true_val=true_val)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     *two_mutual_arrays(dtype_objs=dh.numeric_dtypes,
                        two_shapes=tuples(one_d_shapes, one_d_shapes))
@@ -364,6 +377,7 @@ def test_outer(x1, x2):
 
     assert_exactly_equal(res, true_res)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(rtol=todo)
@@ -372,6 +386,7 @@ def test_pinv(x, kw):
     # res = linalg.pinv(x, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=matrix_shapes),
     kw=kwargs(mode=sampled_from(['reduced', 'complete']))
@@ -407,6 +422,7 @@ def test_qr(x, kw):
     # Check that r is upper-triangular.
     assert_exactly_equal(r, _array_module.triu(r))
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=square_matrix_shapes),
 )
@@ -464,6 +480,7 @@ def x2_shapes(draw):
     x2 = xps.arrays(dtype=xps.floating_dtypes(), shape=x2_shapes())
     return x1, x2
 
+@pytest.mark.xp_extension('linalg')
 @given(*solve_args())
 def test_solve(x1, x2):
     # TODO: solve() is currently ambiguous, in that some inputs can be
@@ -476,6 +493,7 @@ def test_solve(x1, x2):
     # res = linalg.solve(x1, x2)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=finite_matrices,
     kw=kwargs(full_matrices=booleans())
@@ -503,6 +521,7 @@ def test_svd(x, kw):
         assert u.shape == (*stack, M, K)
         assert vh.shape == (*stack, K, N)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
 )
@@ -519,6 +538,7 @@ def test_tensordot(x1, x2, kw):
     # res = _array_module.tensordot(x1, x2, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(offset=todo)
@@ -536,6 +556,7 @@ def test_vecdot(x1, x2, kw):
     # res = _array_module.vecdot(x1, x2, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(axis=todo, keepdims=todo, ord=todo)

array_api_tests/meta_tests/test_hypothesis_helpers.py renamed to array_api_tests/meta/test_hypothesis_helpers.py

@@ -26,7 +26,18 @@ def extension_module(name):
     if extension_module(n):
         extension_module_names.extend([f'{n}.{i}' for i in getattr(function_stubs, n).__all__])
 
-all_names = function_stubs.__all__ + extension_module_names
+
+params = []
+for name in function_stubs.__all__:
+    marks = []
+    if extension_module(name):
+        marks.append(pytest.mark.xp_extension(name))
+    params.append(pytest.param(name, marks=marks))
+for name in extension_module_names:
+    ext = name.split('.')[0]
+    mark = pytest.mark.xp_extension(ext)
+    params.append(pytest.param(name, marks=[mark]))
+
 
 def array_method(name):
     return stub_module(name) == 'array_object'
@@ -130,7 +141,7 @@ def example_argument(arg, func_name, dtype):
     else:
         raise RuntimeError(f"Don't know how to test argument {arg}. Please update test_signatures.py")
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_has_names(name):
     if extension_module(name):
         assert hasattr(mod, name), f'{mod_name} is missing the {name} extension'
@@ -146,7 +157,7 @@ def test_has_names(name):
     else:
         assert hasattr(mod, name), f"{mod_name} is missing the {function_category(name)} function {name}()"
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_function_positional_args(name):
     # Note: We can't actually test that positional arguments are
     # positional-only, as that would require knowing the argument name and
@@ -224,7 +235,7 @@ def test_function_positional_args(name):
             # NumPy ufuncs raise ValueError instead of TypeError
             raises((TypeError, ValueError), lambda: mod_func(*args[:n]), f"{name}() should not accept {n} positional arguments")
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_function_keyword_only_args(name):
     if extension_module(name):
         return