From 9634d04dd299500a36baab6bfbca17d957ec8be4 Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Thu, 14 Oct 2021 17:10:21 +0100
Subject: [PATCH 01/15] Add custom marker `xp_extension()`

---
 conftest.py | 67 ++++++++++++++++++++++++++++++++---------------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/conftest.py b/conftest.py
index cd188b3d..816148ba 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,30 +1,43 @@
 from hypothesis import settings
+from pytest import mark
+
+
+settings.register_profile('xp_default', deadline=800)
 
-# Add a --hypothesis-max-examples flag to pytest. See
-# https://github.com/HypothesisWorks/hypothesis/issues/2434#issuecomment-630309150
 
 def pytest_addoption(parser):
-    # Add an option to change the Hypothesis max_examples setting.
+    # Enable extensions
     parser.addoption(
-        "--hypothesis-max-examples",
-        "--max-examples",
-        action="store",
+        '--ext',
+        '--extensions',
+        nargs='+',
+        default=[],
+        help='enable testing for Array API extensions',
+    )
+    # Hypothesis max examples
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2434
+    parser.addoption(
+        '--hypothesis-max-examples',
+        '--max-examples',
+        action='store',
         default=None,
-        help="set the Hypothesis max_examples setting",
+        help='set the Hypothesis max_examples setting',
     )
-
-    # Add an option to disable the Hypothesis deadline
+    # Hypothesis deadline
     parser.addoption(
-        "--hypothesis-disable-deadline",
-        "--disable-deadline",
-        action="store_true",
-        help="disable the Hypothesis deadline",
+        '--hypothesis-disable-deadline',
+        '--disable-deadline',
+        action='store_true',
+        help='disable the Hypothesis deadline',
     )
 
 
 def pytest_configure(config):
-    # Set Hypothesis max_examples.
-    hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
+    config.addinivalue_line(
+        'markers', 'xp_extension(ext): tests an Array API extension'
+    )
+    # Configure Hypothesis
+    hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
     disable_deadline = config.getoption('--hypothesis-disable-deadline')
     profile_settings = {}
     if hypothesis_max_examples is not None:
@@ -32,14 +45,16 @@ def pytest_configure(config):
     if disable_deadline is not None:
         profile_settings['deadline'] = None
     if profile_settings:
-        import hypothesis
-
-        hypothesis.settings.register_profile(
-            "array-api-tests-hypothesis-overridden", **profile_settings,
-        )
-
-        hypothesis.settings.load_profile("array-api-tests-hypothesis-overridden")
-
-
-settings.register_profile('array_api_tests_hypothesis_profile', deadline=800)
-settings.load_profile('array_api_tests_hypothesis_profile')
+        settings.register_profile('xp_override', **profile_settings)
+        settings.load_profile('xp_override')
+    else:
+        settings.load_profile('xp_default')
+
+
+def pytest_collection_modifyitems(config, items):
+    exts = config.getoption('--extensions')
+    for item in items:
+        if 'xp_extension' in item.keywords:
+            ext = item.keywords['xp_extension'].args[0]
+            if ext not in exts:
+                item.add_marker(mark.skip(reason=f'{ext} not enabled in --extensions'))

From a40ac214ff38d4e99d9c91373709a6271dcc1317 Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Fri, 15 Oct 2021 14:59:37 +0100
Subject: [PATCH 02/15] Fix `xp_extension()` mark collection, apply to
 `test_signature.py`

---
 conftest.py => array_api_tests/conftest.py | 27 ++++++++++++++++++----
 array_api_tests/meta_tests/test_utils.py   | 14 +++++++++++
 array_api_tests/test_signatures.py         | 19 +++++++++++----
 3 files changed, 51 insertions(+), 9 deletions(-)
 rename conftest.py => array_api_tests/conftest.py (69%)
 create mode 100644 array_api_tests/meta_tests/test_utils.py

diff --git a/conftest.py b/array_api_tests/conftest.py
similarity index 69%
rename from conftest.py
rename to array_api_tests/conftest.py
index 816148ba..ed8cedc7 100644
--- a/conftest.py
+++ b/array_api_tests/conftest.py
@@ -1,6 +1,11 @@
+from functools import lru_cache
+
 from hypothesis import settings
 from pytest import mark
 
+from . import _array_module as xp
+from ._array_module import _UndefinedStub
+
 
 settings.register_profile('xp_default', deadline=800)
 
@@ -9,10 +14,10 @@ def pytest_addoption(parser):
     # Enable extensions
     parser.addoption(
         '--ext',
-        '--extensions',
+        '--disable-extensions',
         nargs='+',
         default=[],
-        help='enable testing for Array API extensions',
+        help='disable testing for Array API extensions',
     )
     # Hypothesis max examples
     # See https://github.com/HypothesisWorks/hypothesis/issues/2434
@@ -51,10 +56,22 @@ def pytest_configure(config):
         settings.load_profile('xp_default')
 
 
+@lru_cache
+def xp_has_ext(ext: str) -> bool:
+    try:
+        return not isinstance(getattr(xp, ext), _UndefinedStub)
+    except AttributeError:
+        return False
+
+
 def pytest_collection_modifyitems(config, items):
-    exts = config.getoption('--extensions')
+    disabled_exts = config.getoption('--disable-extensions')
     for item in items:
         if 'xp_extension' in item.keywords:
             ext = item.keywords['xp_extension'].args[0]
-            if ext not in exts:
-                item.add_marker(mark.skip(reason=f'{ext} not enabled in --extensions'))
+            if ext in disabled_exts:
+                item.add_marker(
+                    mark.skip(reason=f'{ext} disabled in --disable-extensions')
+                )
+            elif not xp_has_ext(ext):
+                item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
diff --git a/array_api_tests/meta_tests/test_utils.py b/array_api_tests/meta_tests/test_utils.py
new file mode 100644
index 00000000..e79900bf
--- /dev/null
+++ b/array_api_tests/meta_tests/test_utils.py
@@ -0,0 +1,14 @@
+from ..test_signatures import extension_module
+from ..conftest import xp_has_ext
+
+
+def test_extension_module_is_extension():
+    assert extension_module('linalg')
+
+
+def test_extension_func_is_not_extension():
+    assert not extension_module('linalg.cross')
+
+
+def test_xp_has_ext():
+    assert not xp_has_ext('nonexistent_extension')
diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py
index e8106985..436701ab 100644
--- a/array_api_tests/test_signatures.py
+++ b/array_api_tests/test_signatures.py
@@ -26,7 +26,18 @@ def extension_module(name):
     if extension_module(n):
         extension_module_names.extend([f'{n}.{i}' for i in getattr(function_stubs, n).__all__])
 
-all_names = function_stubs.__all__ + extension_module_names
+
+params = []
+for name in function_stubs.__all__:
+    marks = []
+    if extension_module(name):
+        marks.append(pytest.mark.xp_extension(name))
+    params.append(pytest.param(name, marks=marks))
+for name in extension_module_names:
+    ext = name.split('.')[0]
+    mark = pytest.mark.xp_extension(ext)
+    params.append(pytest.param(name, marks=[mark]))
+
 
 def array_method(name):
     return stub_module(name) == 'array_object'
@@ -130,7 +141,7 @@ def example_argument(arg, func_name, dtype):
     else:
         raise RuntimeError(f"Don't know how to test argument {arg}. Please update test_signatures.py")
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_has_names(name):
     if extension_module(name):
         assert hasattr(mod, name), f'{mod_name} is missing the {name} extension'
@@ -146,7 +157,7 @@ def test_has_names(name):
     else:
         assert hasattr(mod, name), f"{mod_name} is missing the {function_category(name)} function {name}()"
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_function_positional_args(name):
     # Note: We can't actually test that positional arguments are
     # positional-only, as that would require knowing the argument name and
@@ -224,7 +235,7 @@ def test_function_positional_args(name):
             # NumPy ufuncs raise ValueError instead of TypeError
             raises((TypeError, ValueError), lambda: mod_func(*args[:n]), f"{name}() should not accept {n} positional arguments")
 
-@pytest.mark.parametrize('name', all_names)
+@pytest.mark.parametrize('name', params)
 def test_function_keyword_only_args(name):
     if extension_module(name):
         return

From 49ed0c0f6be1126a38804766c54024a971f3daba Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Fri, 15 Oct 2021 16:43:52 +0100
Subject: [PATCH 03/15] Fix `xp_extension()` collection, apply to
 `test_linalg.py`

---
 array_api_tests/conftest.py    | 27 ++++++++++++++++-----------
 array_api_tests/test_linalg.py |  5 +++++
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/array_api_tests/conftest.py b/array_api_tests/conftest.py
index ed8cedc7..e04a0b28 100644
--- a/array_api_tests/conftest.py
+++ b/array_api_tests/conftest.py
@@ -14,10 +14,10 @@ def pytest_addoption(parser):
     # Enable extensions
     parser.addoption(
         '--ext',
-        '--disable-extensions',
+        '--disable-extension',
         nargs='+',
         default=[],
-        help='disable testing for Array API extensions',
+        help='disable testing for Array API extension(s)',
     )
     # Hypothesis max examples
     # See https://github.com/HypothesisWorks/hypothesis/issues/2434
@@ -65,13 +65,18 @@ def xp_has_ext(ext: str) -> bool:
 
 
 def pytest_collection_modifyitems(config, items):
-    disabled_exts = config.getoption('--disable-extensions')
+    disabled_exts = config.getoption('--disable-extension')
     for item in items:
-        if 'xp_extension' in item.keywords:
-            ext = item.keywords['xp_extension'].args[0]
-            if ext in disabled_exts:
-                item.add_marker(
-                    mark.skip(reason=f'{ext} disabled in --disable-extensions')
-                )
-            elif not xp_has_ext(ext):
-                item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
+        try:
+            ext_mark = next(
+                mark for mark in item.iter_markers() if mark.name == 'xp_extension'
+            )
+        except StopIteration:
+            continue
+        ext = ext_mark.args[0]
+        if ext in disabled_exts:
+            item.add_marker(
+                mark.skip(reason=f'{ext} disabled in --disable-extensions')
+            )
+        elif not xp_has_ext(ext):
+            item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
diff --git a/array_api_tests/test_linalg.py b/array_api_tests/test_linalg.py
index afd32f5c..e911db7a 100644
--- a/array_api_tests/test_linalg.py
+++ b/array_api_tests/test_linalg.py
@@ -13,6 +13,7 @@
 
 """
 
+import pytest
 from hypothesis import assume, given
 from hypothesis.strategies import (booleans, composite, none, tuples, integers,
                                    shared, sampled_from)
@@ -33,6 +34,10 @@
 from . import _array_module
 from ._array_module import linalg
 
+
+pytestmark = [pytest.mark.xp_extension('linalg')]
+
+
 # Standin strategy for not yet implemented tests
 todo = none()
 

From 62ded604d06bd8ce8f292db9a6e990080895882b Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Fri, 15 Oct 2021 16:44:35 +0100
Subject: [PATCH 04/15] Rename `meta_tests` to `meta`

---
 array_api_tests/{meta_tests => meta}/__init__.py                | 0
 array_api_tests/{meta_tests => meta}/test_array_helpers.py      | 0
 array_api_tests/{meta_tests => meta}/test_hypothesis_helpers.py | 0
 array_api_tests/{meta_tests => meta}/test_utils.py              | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename array_api_tests/{meta_tests => meta}/__init__.py (100%)
 rename array_api_tests/{meta_tests => meta}/test_array_helpers.py (100%)
 rename array_api_tests/{meta_tests => meta}/test_hypothesis_helpers.py (100%)
 rename array_api_tests/{meta_tests => meta}/test_utils.py (100%)

diff --git a/array_api_tests/meta_tests/__init__.py b/array_api_tests/meta/__init__.py
similarity index 100%
rename from array_api_tests/meta_tests/__init__.py
rename to array_api_tests/meta/__init__.py
diff --git a/array_api_tests/meta_tests/test_array_helpers.py b/array_api_tests/meta/test_array_helpers.py
similarity index 100%
rename from array_api_tests/meta_tests/test_array_helpers.py
rename to array_api_tests/meta/test_array_helpers.py
diff --git a/array_api_tests/meta_tests/test_hypothesis_helpers.py b/array_api_tests/meta/test_hypothesis_helpers.py
similarity index 100%
rename from array_api_tests/meta_tests/test_hypothesis_helpers.py
rename to array_api_tests/meta/test_hypothesis_helpers.py
diff --git a/array_api_tests/meta_tests/test_utils.py b/array_api_tests/meta/test_utils.py
similarity index 100%
rename from array_api_tests/meta_tests/test_utils.py
rename to array_api_tests/meta/test_utils.py

From f8d12dec4c8b8ffb28036186ed38768857c8b859 Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Fri, 15 Oct 2021 17:05:09 +0100
Subject: [PATCH 05/15] Remove redundant linalg xfail

Should be skipped by new `xp_extension()` mark
---
 .github/workflows/numpy.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml
index 7e1eed7d..7501f853 100644
--- a/.github/workflows/numpy.yml
+++ b/.github/workflows/numpy.yml
@@ -31,8 +31,6 @@ jobs:
             "array_api_tests/test_creation_functions.py::test_linspace",
             # einsum is not yet completed in the spec
             "array_api_tests/test_signatures.py::test_has_names[einsum]",
-            # The linalg extension is not yet implemented in NumPy
-            "array_api_tests/test_signatures.py::test_has_names[linalg]",
             # dlpack support is not yet implemented in NumPy. https://github.com/numpy/numpy/pull/19083
             "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]",
             "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]",

From 15640a2fc0ee249f6bbe1cd775146365da77b645 Mon Sep 17 00:00:00 2001
From: Matthew Barber <quitesimplymatt@gmail.com>
Date: Fri, 15 Oct 2021 18:21:08 +0100
Subject: [PATCH 06/15] Split `conftest.py`

---
 array_api_tests/conftest.py | 38 +------------------------------------
 conftest.py                 | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 37 deletions(-)
 create mode 100644 conftest.py

diff --git a/array_api_tests/conftest.py b/array_api_tests/conftest.py
index e04a0b28..fd1ceb1e 100644
--- a/array_api_tests/conftest.py
+++ b/array_api_tests/conftest.py
@@ -1,17 +1,12 @@
 from functools import lru_cache
 
-from hypothesis import settings
 from pytest import mark
 
 from . import _array_module as xp
 from ._array_module import _UndefinedStub
 
 
-settings.register_profile('xp_default', deadline=800)
-
-
 def pytest_addoption(parser):
-    # Enable extensions
     parser.addoption(
         '--ext',
         '--disable-extension',
@@ -19,41 +14,12 @@ def pytest_addoption(parser):
         default=[],
         help='disable testing for Array API extension(s)',
     )
-    # Hypothesis max examples
-    # See https://github.com/HypothesisWorks/hypothesis/issues/2434
-    parser.addoption(
-        '--hypothesis-max-examples',
-        '--max-examples',
-        action='store',
-        default=None,
-        help='set the Hypothesis max_examples setting',
-    )
-    # Hypothesis deadline
-    parser.addoption(
-        '--hypothesis-disable-deadline',
-        '--disable-deadline',
-        action='store_true',
-        help='disable the Hypothesis deadline',
-    )
 
 
 def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'xp_extension(ext): tests an Array API extension'
     )
-    # Configure Hypothesis
-    hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
-    disable_deadline = config.getoption('--hypothesis-disable-deadline')
-    profile_settings = {}
-    if hypothesis_max_examples is not None:
-        profile_settings['max_examples'] = int(hypothesis_max_examples)
-    if disable_deadline is not None:
-        profile_settings['deadline'] = None
-    if profile_settings:
-        settings.register_profile('xp_override', **profile_settings)
-        settings.load_profile('xp_override')
-    else:
-        settings.load_profile('xp_default')
 
 
 @lru_cache
@@ -75,8 +41,6 @@ def pytest_collection_modifyitems(config, items):
             continue
         ext = ext_mark.args[0]
         if ext in disabled_exts:
-            item.add_marker(
-                mark.skip(reason=f'{ext} disabled in --disable-extensions')
-            )
+            item.add_marker(mark.skip(reason=f'{ext} disabled in --disable-extensions'))
         elif not xp_has_ext(ext):
             item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
diff --git a/conftest.py b/conftest.py
new file mode 100644
index 00000000..b967858e
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,37 @@
+from hypothesis import settings
+
+settings.register_profile('xp_default', deadline=800)
+
+
+def pytest_addoption(parser):
+    # Hypothesis max examples
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2434
+    parser.addoption(
+        '--hypothesis-max-examples',
+        '--max-examples',
+        action='store',
+        default=None,
+        help='set the Hypothesis max_examples setting',
+    )
+    # Hypothesis deadline
+    parser.addoption(
+        '--hypothesis-disable-deadline',
+        '--disable-deadline',
+        action='store_true',
+        help='disable the Hypothesis deadline',
+    )
+
+
+def pytest_configure(config):
+    hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
+    disable_deadline = config.getoption('--hypothesis-disable-deadline')
+    profile_settings = {}
+    if hypothesis_max_examples is not None:
+        profile_settings['max_examples'] = int(hypothesis_max_examples)
+    if disable_deadline is not None:
+        profile_settings['deadline'] = None
+    if profile_settings:
+        settings.register_profile('xp_override', **profile_settings)
+        settings.load_profile('xp_override')
+    else:
+        settings.load_profile('xp_default')

From 2d3835c4b9748e49a68cc1e27822cdfd11bc2fb4 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Tue, 19 Oct 2021 10:57:06 +0100
Subject: [PATCH 07/15] Remove `xp_extension()` for top-level linalg tests

---
 array_api_tests/test_linalg.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/array_api_tests/test_linalg.py b/array_api_tests/test_linalg.py
index e911db7a..421eeacd 100644
--- a/array_api_tests/test_linalg.py
+++ b/array_api_tests/test_linalg.py
@@ -35,9 +35,6 @@
 from ._array_module import linalg
 
 
-pytestmark = [pytest.mark.xp_extension('linalg')]
-
-
 # Standin strategy for not yet implemented tests
 todo = none()
 
@@ -79,6 +76,7 @@ def _test_namedtuple(res, fields, func_name):
         assert hasattr(res, field), f"{func_name}() result namedtuple doesn't have the '{field}' field"
         assert res[i] is getattr(res, field), f"{func_name}() result namedtuple '{field}' field is not in position {i}"
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=positive_definite_matrices(),
     kw=kwargs(upper=booleans())
@@ -126,6 +124,7 @@ def cross_args(draw, dtype_objects=dh.numeric_dtypes):
     )
     return draw(arrays1), draw(arrays2), kw
 
+@pytest.mark.xp_extension('linalg')
 @given(
     cross_args()
 )
@@ -164,6 +163,7 @@ def test_cross(x1_x2_kw):
             ], dtype=res.dtype)
         assert_exactly_equal(res_stack, exact_cross)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=square_matrix_shapes),
 )
@@ -177,6 +177,7 @@ def test_det(x):
 
     # TODO: Test that res actually corresponds to the determinant of x
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=dtypes, shape=matrix_shapes),
     # offset may produce an overflow if it is too large. Supporting offsets
@@ -211,6 +212,7 @@ def true_diag(x_stack):
 
     _test_stacks(linalg.diagonal, x, **kw, res=res, dims=1, true_val=true_diag)
 
+@pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))
 def test_eigh(x):
     res = linalg.eigh(x)
@@ -234,6 +236,7 @@ def test_eigh(x):
     # TODO: Test that res actually corresponds to the eigenvalues and
     # eigenvectors of x
 
+@pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))
 def test_eigvalsh(x):
     res = linalg.eigvalsh(x)
@@ -247,6 +250,7 @@ def test_eigvalsh(x):
 
     # TODO: Test that res actually corresponds to the eigenvalues of x
 
+@pytest.mark.xp_extension('linalg')
 @given(x=invertible_matrices())
 def test_inv(x):
     res = linalg.inv(x)
@@ -291,6 +295,7 @@ def test_matmul(x1, x2):
         assert res.shape == stack_shape + (x1.shape[-2], x2.shape[-1])
         _test_stacks(_array_module.matmul, x1, x2, res=res)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(axis=todo, keepdims=todo, ord=todo)
@@ -300,6 +305,7 @@ def test_matrix_norm(x, kw):
     pass
 
 matrix_power_n = shared(integers(-1000, 1000), key='matrix_power n')
+@pytest.mark.xp_extension('linalg')
 @given(
     # Generate any square matrix if n >= 0 but only invertible matrices if n < 0
     x=matrix_power_n.flatmap(lambda n: invertible_matrices() if n < 0 else
@@ -321,6 +327,7 @@ def test_matrix_power(x, n):
     func = lambda x: linalg.matrix_power(x, n)
     _test_stacks(func, x, res=res, true_val=true_val)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(rtol=todo)
@@ -346,6 +353,7 @@ def test_matrix_transpose(x):
 
     _test_stacks(_array_module.matrix_transpose, x, res=res, true_val=true_val)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     *two_mutual_arrays(dtype_objs=dh.numeric_dtypes,
                        two_shapes=tuples(one_d_shapes, one_d_shapes))
@@ -369,6 +377,7 @@ def test_outer(x1, x2):
 
     assert_exactly_equal(res, true_res)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(rtol=todo)
@@ -377,6 +386,7 @@ def test_pinv(x, kw):
     # res = linalg.pinv(x, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=matrix_shapes),
     kw=kwargs(mode=sampled_from(['reduced', 'complete']))
@@ -412,6 +422,7 @@ def test_qr(x, kw):
     # Check that r is upper-triangular.
     assert_exactly_equal(r, _array_module.triu(r))
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=square_matrix_shapes),
 )
@@ -469,6 +480,7 @@ def x2_shapes(draw):
     x2 = xps.arrays(dtype=xps.floating_dtypes(), shape=x2_shapes())
     return x1, x2
 
+@pytest.mark.xp_extension('linalg')
 @given(*solve_args())
 def test_solve(x1, x2):
     # TODO: solve() is currently ambiguous, in that some inputs can be
@@ -481,6 +493,7 @@ def test_solve(x1, x2):
     # res = linalg.solve(x1, x2)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=finite_matrices,
     kw=kwargs(full_matrices=booleans())
@@ -508,6 +521,7 @@ def test_svd(x, kw):
         assert u.shape == (*stack, M, K)
         assert vh.shape == (*stack, K, N)
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
 )
@@ -524,6 +538,7 @@ def test_tensordot(x1, x2, kw):
     # res = _array_module.tensordot(x1, x2, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(offset=todo)
@@ -541,6 +556,7 @@ def test_vecdot(x1, x2, kw):
     # res = _array_module.vecdot(x1, x2, **kw)
     pass
 
+@pytest.mark.xp_extension('linalg')
 @given(
     x=xps.arrays(dtype=xps.floating_dtypes(), shape=shapes),
     kw=kwargs(axis=todo, keepdims=todo, ord=todo)

From d9d8bc18ae4fe3719414bce57935421d60fceab1 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 08:55:28 +0100
Subject: [PATCH 08/15] Remove `--ext` shorthand option

---
 array_api_tests/conftest.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/array_api_tests/conftest.py b/array_api_tests/conftest.py
index fd1ceb1e..a6404607 100644
--- a/array_api_tests/conftest.py
+++ b/array_api_tests/conftest.py
@@ -8,7 +8,6 @@
 
 def pytest_addoption(parser):
     parser.addoption(
-        '--ext',
         '--disable-extension',
         nargs='+',
         default=[],

From 39b11d1bbf34bff0d6140e52b953fe76c431af86 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 08:59:53 +0100
Subject: [PATCH 09/15] Remove numpy requirement in installation instructions

---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index bb641538..9a1b420e 100644
--- a/README.md
+++ b/README.md
@@ -12,17 +12,17 @@ specification that are not yet tested here.
 
 To run the tests, first install the testing dependencies
 
-    pip install pytest hypothesis numpy
+    pip install pytest hypothesis
 
 or
 
-    conda install pytest hypothesis numpy
+    conda install pytest hypothesis
 
-as well as the array libraries that you want to test. (Note, in the future,
-NumPy will be removed as a dependency on the test suite). To run the tests,
-you need to set the array library that is to be tested. There are two ways to
-do this. One way is to set the `ARRAY_API_TESTS_MODULE` environment variable.
-For example
+as well as the array libraries that you want to test. 
+
+To run the tests, you need to set the array library that is to be tested. There
+are two ways to do this. One way is to set the `ARRAY_API_TESTS_MODULE`
+environment variable. For example
 
     ARRAY_API_TESTS_MODULE=numpy pytest
 

From 0d19396cefc42b83d85de010c6221abcbc33a3e7 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 09:10:04 +0100
Subject: [PATCH 10/15] Add `--enable-extension` option

---
 array_api_tests/conftest.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/array_api_tests/conftest.py b/array_api_tests/conftest.py
index a6404607..ae58b6d7 100644
--- a/array_api_tests/conftest.py
+++ b/array_api_tests/conftest.py
@@ -9,10 +9,18 @@
 def pytest_addoption(parser):
     parser.addoption(
         '--disable-extension',
+        metavar='ext',
         nargs='+',
         default=[],
         help='disable testing for Array API extension(s)',
     )
+    parser.addoption(
+        '--enable-extension',
+        metavar='ext',
+        nargs='+',
+        default=[],
+        help='enable testing for Array API extension(s)',
+    )
 
 
 def pytest_configure(config):
@@ -31,6 +39,10 @@ def xp_has_ext(ext: str) -> bool:
 
 def pytest_collection_modifyitems(config, items):
     disabled_exts = config.getoption('--disable-extension')
+    enabled_exts = config.getoption('--enable-extension')
+    for ext in disabled_exts:
+        if ext in enabled_exts:
+            raise ValueError(f'{ext=} both enabled and disabled')
     for item in items:
         try:
             ext_mark = next(
@@ -41,5 +53,5 @@ def pytest_collection_modifyitems(config, items):
         ext = ext_mark.args[0]
         if ext in disabled_exts:
             item.add_marker(mark.skip(reason=f'{ext} disabled in --disable-extensions'))
-        elif not xp_has_ext(ext):
+        elif not ext in enabled_exts and not xp_has_ext(ext):
             item.add_marker(mark.skip(reason=f'{ext} not found in array module'))

From c849cf8f57d06793c7aee218bec2822c9e5631e5 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 09:29:29 +0100
Subject: [PATCH 11/15] Document enable/disable ext options

---
 README.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/README.md b/README.md
index 9a1b420e..b88116ef 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,8 @@ specification that are not yet tested here.
 
 ## Running the tests
 
+### Setup
+
 To run the tests, first install the testing dependencies
 
     pip install pytest hypothesis
@@ -20,6 +22,8 @@ or
 
 as well as the array libraries that you want to test. 
 
+### Specifying the array module
+
 To run the tests, you need to set the array library that is to be tested. There
 are two ways to do this. One way is to set the `ARRAY_API_TESTS_MODULE`
 environment variable. For example
@@ -41,6 +45,29 @@ import numpy as array_module
 
 (replacing `numpy` with the array module namespace to be tested).
 
+### Run pytest
+
+Simply run the `pytest` command to run the whole test suite.
+
+```bash
+pytest
+```
+
+The test suite tries to logically organise its tests so you can find specific
+test cases whilst developing something in particular. So to avoid running the
+rather slow complete suite, you can specify particular test cases like any other
+test suite.
+
+```bash
+pytest array_api_tests/test_creation_functions.py::test_zeros
+```
+
+By default, tests for the optional Array API extensions such as
+[`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
+will be skipped if not present in the specified array module. You can purposely
+skip testing extension(s) via the `--disable-extension` option, and likewise
+purposely test them via the `--enable-extension` option.
+
 ## Notes on Interpreting Errors
 
 - Some tests cannot be run unless other tests pass first. This is because very

From a860c46da14eb749187018b815043dfabf849c60 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 10:39:39 +0100
Subject: [PATCH 12/15] Read `xfails.txt` for xfailing tests, update NumPy
 workflow

And merges both `conftest.py` files together
---
 .github/workflows/numpy.yml        | 58 ++++++++++++-------------
 array_api_tests/conftest.py        | 57 ------------------------
 array_api_tests/meta/test_utils.py |  5 ---
 conftest.py                        | 69 ++++++++++++++++++++++++++++++
 4 files changed, 96 insertions(+), 93 deletions(-)
 delete mode 100644 array_api_tests/conftest.py

diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml
index 7501f853..ee0a70f7 100644
--- a/.github/workflows/numpy.yml
+++ b/.github/workflows/numpy.yml
@@ -22,39 +22,35 @@ jobs:
         python -m pip install git+https://github.com/numpy/numpy
         python -m pip install -r requirements.txt
     - name: Run the test suite
+      env:
+        ARRAY_API_TESTS_MODULE: numpy.array_api
       run: |
         # Mark some known issues as XFAIL
-        cat << EOF >> conftest.py
+        cat << EOF >> xfails.txt
+
+        # https://github.com/numpy/numpy/issues/18881
+        array_api_tests/test_creation_functions.py::test_linspace
+        # einsum is not yet completed in the spec
+        array_api_tests/test_signatures.py::test_has_names[einsum]
+        # dlpack support is not yet implemented in NumPy
+        # See https://github.com/numpy/numpy/pull/19083
+        array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]
+        array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]
+        array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]
+        array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]
+        # Updates to the spec since the last change to numpy.array_api
+        # These will fail until NumPy is updated
+        array_api_tests/test_signatures.py::test_has_names[__index__]
+        array_api_tests/test_signatures.py::test_has_names[to_device]
+        array_api_tests/test_signatures.py::test_has_names[mT]
+        array_api_tests/test_signatures.py::test_has_names[tril]
+        array_api_tests/test_signatures.py::test_has_names[triu]
+        array_api_tests/test_signatures.py::test_has_names[matrix_transpose]
+        array_api_tests/test_signatures.py::test_has_names[permute_dims]
+        array_api_tests/test_signatures.py::test_function_positional_args[__index__]
+        array_api_tests/test_signatures.py::test_function_keyword_only_args[prod]
+        array_api_tests/test_signatures.py::test_function_keyword_only_args[sum]
 
-        names_to_be_xfailed = (
-            # https://github.com/numpy/numpy/issues/18881
-            "array_api_tests/test_creation_functions.py::test_linspace",
-            # einsum is not yet completed in the spec
-            "array_api_tests/test_signatures.py::test_has_names[einsum]",
-            # dlpack support is not yet implemented in NumPy. https://github.com/numpy/numpy/pull/19083
-            "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]",
-
-            # Updates to the spec since the last change to numpy.array_api.
-            # These will fail until NumPy is updated.
-            "array_api_tests/test_signatures.py::test_has_names[__index__]",
-            "array_api_tests/test_signatures.py::test_has_names[to_device]",
-            "array_api_tests/test_signatures.py::test_has_names[mT]",
-            "array_api_tests/test_signatures.py::test_has_names[tril]",
-            "array_api_tests/test_signatures.py::test_has_names[triu]",
-            "array_api_tests/test_signatures.py::test_has_names[matrix_transpose]",
-            "array_api_tests/test_signatures.py::test_has_names[permute_dims]",
-            "array_api_tests/test_signatures.py::test_function_positional_args[__index__]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[prod]",
-            "array_api_tests/test_signatures.py::test_function_keyword_only_args[sum]",
-        )
-
-        def pytest_collection_modifyitems(config, items):
-            for item in items:
-                if item.nodeid in names_to_be_xfailed:
-                    item.add_marker("xfail")
         EOF
 
-        ARRAY_API_TESTS_MODULE=numpy.array_api pytest -v -rxXfE
+        pytest -v -rxXfE
diff --git a/array_api_tests/conftest.py b/array_api_tests/conftest.py
deleted file mode 100644
index ae58b6d7..00000000
--- a/array_api_tests/conftest.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from functools import lru_cache
-
-from pytest import mark
-
-from . import _array_module as xp
-from ._array_module import _UndefinedStub
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        '--disable-extension',
-        metavar='ext',
-        nargs='+',
-        default=[],
-        help='disable testing for Array API extension(s)',
-    )
-    parser.addoption(
-        '--enable-extension',
-        metavar='ext',
-        nargs='+',
-        default=[],
-        help='enable testing for Array API extension(s)',
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        'markers', 'xp_extension(ext): tests an Array API extension'
-    )
-
-
-@lru_cache
-def xp_has_ext(ext: str) -> bool:
-    try:
-        return not isinstance(getattr(xp, ext), _UndefinedStub)
-    except AttributeError:
-        return False
-
-
-def pytest_collection_modifyitems(config, items):
-    disabled_exts = config.getoption('--disable-extension')
-    enabled_exts = config.getoption('--enable-extension')
-    for ext in disabled_exts:
-        if ext in enabled_exts:
-            raise ValueError(f'{ext=} both enabled and disabled')
-    for item in items:
-        try:
-            ext_mark = next(
-                mark for mark in item.iter_markers() if mark.name == 'xp_extension'
-            )
-        except StopIteration:
-            continue
-        ext = ext_mark.args[0]
-        if ext in disabled_exts:
-            item.add_marker(mark.skip(reason=f'{ext} disabled in --disable-extensions'))
-        elif not ext in enabled_exts and not xp_has_ext(ext):
-            item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
diff --git a/array_api_tests/meta/test_utils.py b/array_api_tests/meta/test_utils.py
index e79900bf..f4d7cf1f 100644
--- a/array_api_tests/meta/test_utils.py
+++ b/array_api_tests/meta/test_utils.py
@@ -1,5 +1,4 @@
 from ..test_signatures import extension_module
-from ..conftest import xp_has_ext
 
 
 def test_extension_module_is_extension():
@@ -8,7 +7,3 @@ def test_extension_module_is_extension():
 
 def test_extension_func_is_not_extension():
     assert not extension_module('linalg.cross')
-
-
-def test_xp_has_ext():
-    assert not xp_has_ext('nonexistent_extension')
diff --git a/conftest.py b/conftest.py
index b967858e..74398f3b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,13 @@
+from functools import lru_cache
+from pathlib import Path
+
+from pytest import mark
 from hypothesis import settings
 
+from array_api_tests import _array_module as xp
+from array_api_tests._array_module import _UndefinedStub
+
+
 settings.register_profile('xp_default', deadline=800)
 
 
@@ -20,9 +28,28 @@ def pytest_addoption(parser):
         action='store_true',
         help='disable the Hypothesis deadline',
     )
+    # enable/disable extensions
+    parser.addoption(
+        '--enable-extension',
+        metavar='ext',
+        nargs='+',
+        default=[],
+        help='enable testing for Array API extension(s)',
+    )
+    parser.addoption(
+        '--disable-extension',
+        metavar='ext',
+        nargs='+',
+        default=[],
+        help='disable testing for Array API extension(s)',
+    )
 
 
 def pytest_configure(config):
+    config.addinivalue_line(
+        'markers', 'xp_extension(ext): tests an Array API extension'
+    )
+    # Hypothesis
     hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
     disable_deadline = config.getoption('--hypothesis-disable-deadline')
     profile_settings = {}
@@ -35,3 +62,45 @@ def pytest_configure(config):
         settings.load_profile('xp_override')
     else:
         settings.load_profile('xp_default')
+
+
+@lru_cache
+def xp_has_ext(ext: str) -> bool:
+    try:
+        return not isinstance(getattr(xp, ext), _UndefinedStub)
+    except AttributeError:
+        return False
+
+
+xfail_ids = []
+xfails_path = Path(__file__).parent / 'xfails.txt'
+if xfails_path.exists():
+    with open(xfails_path) as f:
+        for line in f:
+            if line.startswith('test'):
+                id_ = line.strip('\n')
+                xfail_ids.append(id_)
+
+
+def pytest_collection_modifyitems(config, items):
+    enabled_exts = config.getoption('--enable-extension')
+    disabled_exts = config.getoption('--disable-extension')
+    for ext in enabled_exts:
+        if ext in disabled_exts:
+            raise ValueError(f'{ext=} both enabled and disabled')
+    for item in items:
+        # enable/disable extensions
+        try:
+            ext_mark = next(m for m in item.iter_markers() if m.name == 'xp_extension')
+            ext = ext_mark.args[0]
+            if ext in disabled_exts:
+                item.add_marker(
+                    mark.skip(reason=f'{ext} disabled in --disable-extensions')
+                )
+            elif not ext in enabled_exts and not xp_has_ext(ext):
+                item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
+        except StopIteration:
+            pass
+        # workflow xfail_ids
+        if item.nodeid in xfail_ids:
+            item.add_marker(mark.xfail(reason='xfails.txt'))

From 1e80b572f578a689bc94211649cd8a53a22de1ce Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Wed, 20 Oct 2021 20:53:34 +0100
Subject: [PATCH 13/15] Fix xfail collection

---
 conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 74398f3b..804bab29 100644
--- a/conftest.py
+++ b/conftest.py
@@ -77,7 +77,7 @@ def xp_has_ext(ext: str) -> bool:
 if xfails_path.exists():
     with open(xfails_path) as f:
         for line in f:
-            if line.startswith('test'):
+            if line.startswith('array_api_tests'):
                 id_ = line.strip('\n')
                 xfail_ids.append(id_)
 

From 5542a0bd887014467cfb216186fc66232082e7d6 Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Thu, 21 Oct 2021 09:49:58 +0100
Subject: [PATCH 14/15] Improve README updates

---
 README.md | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index b88116ef..c8d30717 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ as well as the array libraries that you want to test.
 
 To run the tests, you need to set the array library that is to be tested. There
 are two ways to do this. One way is to set the `ARRAY_API_TESTS_MODULE`
-environment variable. For example
+environment variable. For example you can set it when running `pytest`
 
     ARRAY_API_TESTS_MODULE=numpy pytest
 
@@ -39,34 +39,20 @@ array_module = None
 
 to
 
-```
+```py
 import numpy as array_module
 ```
 
 (replacing `numpy` with the array module namespace to be tested).
 
-### Run pytest
-
-Simply run the `pytest` command to run the whole test suite.
-
-```bash
-pytest
-```
+### Specifying test cases
 
 The test suite tries to logically organise its tests so you can find specific
 test cases whilst developing something in particular. So to avoid running the
 rather slow complete suite, you can specify particular test cases like any other
 test suite.
 
-```bash
-pytest array_api_tests/test_creation_functions.py::test_zeros
-```
-
-By default, tests for the optional Array API extensions such as
-[`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
-will be skipped if not present in the specified array module. You can purposely
-skip testing extension(s) via the `--disable-extension` option, and likewise
-purposely test them via the `--enable-extension` option.
+    pytest array_api_tests/test_creation_functions.py::test_zeros
 
 ## Notes on Interpreting Errors
 
@@ -93,6 +79,12 @@ purposely test them via the `--enable-extension` option.
 
 ## Configuring Tests
 
+By default, tests for the optional Array API extensions such as
+[`linalg`](https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html)
+will be skipped if not present in the specified array module. You can purposely
+skip testing extension(s) via the `--disable-extension` option, and likewise
+purposely test them via the `--enable-extension` option.
+
 The tests make heavy use of the
 [Hypothesis](https://hypothesis.readthedocs.io/en/latest/) testing library.
 Hypothesis generates random input values for the tests. You can configure how

From 6e452fe80aa0bbf699370024be1c3e052c10c03b Mon Sep 17 00:00:00 2001
From: Matthew <quitesimplymatt@gmail.com>
Date: Thu, 21 Oct 2021 09:54:49 +0100
Subject: [PATCH 15/15] Update NumPy workflow xfails

---
 .github/workflows/numpy.yml | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml
index ee0a70f7..e4694fb0 100644
--- a/.github/workflows/numpy.yml
+++ b/.github/workflows/numpy.yml
@@ -37,19 +37,10 @@ jobs:
         array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]
         array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]
         array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]
+        array_api_tests/test_signatures.py::test_function_positional_args[to_device]
         array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]
-        # Updates to the spec since the last change to numpy.array_api
-        # These will fail until NumPy is updated
-        array_api_tests/test_signatures.py::test_has_names[__index__]
-        array_api_tests/test_signatures.py::test_has_names[to_device]
-        array_api_tests/test_signatures.py::test_has_names[mT]
-        array_api_tests/test_signatures.py::test_has_names[tril]
-        array_api_tests/test_signatures.py::test_has_names[triu]
-        array_api_tests/test_signatures.py::test_has_names[matrix_transpose]
-        array_api_tests/test_signatures.py::test_has_names[permute_dims]
-        array_api_tests/test_signatures.py::test_function_positional_args[__index__]
-        array_api_tests/test_signatures.py::test_function_keyword_only_args[prod]
-        array_api_tests/test_signatures.py::test_function_keyword_only_args[sum]
+        # floor_divide has an issue related to https://github.com/data-apis/array-api/issues/264
+        array_api_tests/test_elementwise_functions.py::test_floor_divide
 
         EOF