PYTHON-4565 - Use pytest markers for test suite configuration (#1741)

Author: NoahStapp

.evergreen/run-tests.sh

@@ -30,7 +30,7 @@ set -o xtrace
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
-TEST_SUITES="test/ test/asynchronous/"
+TEST_SUITES=""
 TEST_ARGS="${*:1}"
 
 export PIP_QUIET=1  # Quiet by default
@@ -96,7 +96,7 @@ if [ -n "$TEST_LOADBALANCER" ]; then
     export LOAD_BALANCER=1
     export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
     export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
-    export TEST_SUITES="test/test_load_balancer.py"
+    export TEST_SUITES="load_balancer"
 fi
 
 if [ "$SSL" != "nossl" ]; then
@@ -172,7 +172,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then
         export PATH=$CRYPT_SHARED_DIR:$PATH
     fi
     # Only run the encryption tests.
-    TEST_SUITES="test/test_encryption.py"
+    TEST_SUITES="encryption"
 fi
 
 if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
@@ -185,8 +185,7 @@ if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
       echo "MONGODB_URI unexpectedly contains user credentials in FLE test!";
       exit 1
     fi
-
-    TEST_SUITES="test/test_on_demand_csfle.py"
+    TEST_SUITES="csfle"
 fi
 
 if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
@@ -195,36 +194,36 @@ if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
     set +x
     export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
     set -x
-    TEST_SUITES="test/test_index_management.py"
+    TEST_SUITES="index_management"
 fi
 
 if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
-    TEST_SUITES="test/test_data_lake.py"
+    TEST_SUITES="data_lake"
 fi
 
 if [ -n "$TEST_ATLAS" ]; then
-    TEST_SUITES="test/atlas/test_connection.py"
+    TEST_SUITES="atlas"
 fi
 
 if [ -n "$TEST_OCSP" ]; then
     python -m pip install ".[ocsp]"
-    TEST_SUITES="test/ocsp/test_ocsp.py"
+    TEST_SUITES="ocsp"
 fi
 
 if [ -n "$TEST_AUTH_AWS" ]; then
     python -m pip install ".[aws]"
-    TEST_SUITES="test/auth_aws/test_auth_aws.py"
+    TEST_SUITES="auth_aws"
 fi
 
 if [ -n "$TEST_AUTH_OIDC" ]; then
     python -m pip install ".[aws]"
-    TEST_SUITES="test/auth_oidc/test_auth_oidc.py $TEST_ARGS"
+    TEST_SUITES="auth_oidc"
 fi
 
 if [ -n "$PERF_TEST" ]; then
     python -m pip install simplejson
     start_time=$(date +%s)
-    TEST_SUITES="test/performance/perf_test.py"
+    TEST_SUITES="perf"
 fi
 
 echo "Running $AUTH tests over $SSL with python $(which python)"
@@ -254,7 +253,11 @@ PIP_QUIET=0 python -m pip list
 if [ -z "$GREEN_FRAMEWORK" ]; then
     # Use --capture=tee-sys so pytest prints test output inline:
     # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
-    python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_SUITES $TEST_ARGS
+    if [ -z "$TEST_SUITES" ]; then
+      python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_ARGS
+    else
+      python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 -m $TEST_SUITES $TEST_ARGS
+    fi
 else
     python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
 fi

hatch.toml

@@ -42,7 +42,7 @@ features = ["test"]
 test = "pytest -v --durations=5 --maxfail=10 {args}"
 test-eg = "bash ./.evergreen/run-tests.sh {args}"
 test-async = "test test/asynchronous/ {args}"
-test-mockupdb = ["pip install -U git+https://github.com/ajdavis/mongo-mockup-db@master", "test ./test/mockupdb"]
+test-mockupdb = ["pip install -U git+https://github.com/ajdavis/mongo-mockup-db@master", "test -m mockupdb"]
 
 [envs.encryption]
 skip-install = true

pyproject.toml

@@ -70,10 +70,9 @@ zstd = ["requirements/zstd.txt"]
 
 [tool.pytest.ini_options]
 minversion = "7"
-addopts = ["-ra", "--strict-config", "--strict-markers", "--junitxml=xunit-results/TEST-results.xml"]
+addopts = ["-ra", "--strict-config", "--strict-markers", "--junitxml=xunit-results/TEST-results.xml", "-m default"]
 testpaths = ["test"]
 log_cli_level = "INFO"
-norecursedirs = ["test/*"]
 faulthandler_timeout = 1500
 xfail_strict = true
 filterwarnings = [
@@ -96,6 +95,20 @@ filterwarnings = [
     # https://github.com/dateutil/dateutil/issues/1314
     "module:datetime.datetime.utc:DeprecationWarning:dateutil",
 ]
+markers = [
+    "auth_aws: tests that rely on pymongo-auth-aws",
+    "auth_oidc: tests that rely on oidc auth",
+    "ocsp: tests that rely on ocsp",
+    "atlas: tests that rely on atlas",
+    "data_lake: tests that rely on atlas data lake",
+    "perf: benchmark tests",
+    "index_management: index management tests",
+    "csfle: client-side field-level encryption tests",
+    "encryption: encryption tests",
+    "load_balancer: load balancer tests",
+    "mockupdb: tests that rely on mockupdb",
+    "default: default test suite",
+]
 
 [tool.mypy]
 strict = true

test/asynchronous/conftest.py

@@ -12,3 +12,9 @@ async def test_setup_and_teardown():
     await async_setup()
     yield
     await async_teardown()
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("default")

test/atlas/test_connection.py

@@ -20,11 +20,16 @@
 import unittest
 from collections import defaultdict
 
+import pytest
+
 sys.path[0:0] = [""]
 
 import pymongo
 from pymongo.ssl_support import HAS_SNI
 
+pytestmark = pytest.mark.atlas
+
+
 URIS = {
     "ATLAS_REPL": os.environ.get("ATLAS_REPL"),
     "ATLAS_SHRD": os.environ.get("ATLAS_SHRD"),

test/auth_aws/test_auth_aws.py

@@ -20,14 +20,21 @@
 import unittest
 from unittest.mock import patch
 
+import pytest
+
 sys.path[0:0] = [""]
 
-from pymongo_auth_aws import AwsCredential, auth
+try:
+    from pymongo_auth_aws import AwsCredential, auth
+except ImportError:
+    pass
 
 from pymongo import MongoClient
 from pymongo.errors import OperationFailure
 from pymongo.uri_parser import parse_uri
 
+pytestmark = pytest.mark.auth_aws
+
 
 class TestAuthAWS(unittest.TestCase):
     uri: str

test/auth_oidc/test_auth_oidc.py

@@ -25,6 +25,8 @@
 from pathlib import Path
 from typing import Dict
 
+import pytest
+
 sys.path[0:0] = [""]
 
 from test.unified_format import generate_test_classes
@@ -55,6 +57,8 @@
 # Generate unified tests.
 globals().update(generate_test_classes(str(TEST_PATH), module=__name__))
 
+pytestmark = pytest.mark.auth_oidc
+
 
 class OIDCTestBase(unittest.TestCase):
     @classmethod
@@ -96,6 +100,7 @@ def fail_point(self, command_args):
             client.admin.command("configureFailPoint", cmd_on["configureFailPoint"], mode="off")
 
 
+@pytest.mark.auth_oidc
 class TestAuthOIDCHuman(OIDCTestBase):
     uri: str
 

test/conftest.py

@@ -12,3 +12,9 @@ def test_setup_and_teardown():
     setup()
     yield
     teardown()
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("default")

test/mockupdb/operations.py

@@ -15,7 +15,12 @@
 
 from collections import namedtuple
 
-from mockupdb import OpMsgReply, OpReply
+try:
+    from mockupdb import OpMsgReply, OpReply
+
+    _HAVE_MOCKUPDB = True
+except ImportError:
+    _HAVE_MOCKUPDB = False
 
 from pymongo import ReadPreference
 
@@ -51,67 +56,69 @@
 sharded cluster (PYTHON-868).
 """
 
-not_master_reply = OpMsgReply(ok=0, errmsg="not master")
-
-operations = [
-    Operation(
-        "find_one",
-        lambda client: client.db.collection.find_one(),
-        reply={"cursor": {"id": 0, "firstBatch": []}},
-        op_type="may-use-secondary",
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "count_documents",
-        lambda client: client.db.collection.count_documents({}),
-        reply={"n": 1},
-        op_type="may-use-secondary",
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "estimated_document_count",
-        lambda client: client.db.collection.estimated_document_count(),
-        reply={"n": 1},
-        op_type="may-use-secondary",
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "aggregate",
-        lambda client: client.db.collection.aggregate([]),
-        reply={"cursor": {"id": 0, "firstBatch": []}},
-        op_type="may-use-secondary",
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "options",
-        lambda client: client.db.collection.options(),
-        reply={"cursor": {"id": 0, "firstBatch": []}},
-        op_type="must-use-primary",
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "command",
-        lambda client: client.db.command("foo"),
-        reply={"ok": 1},
-        op_type="must-use-primary",  # Ignores client's read preference.
-        not_master=not_master_reply,
-    ),
-    Operation(
-        "secondary command",
-        lambda client: client.db.command("foo", read_preference=ReadPreference.SECONDARY),
-        reply={"ok": 1},
-        op_type="always-use-secondary",
-        not_master=OpReply(ok=0, errmsg="node is recovering"),
-    ),
-    Operation(
-        "listIndexes",
-        lambda client: client.db.collection.index_information(),
-        reply={"cursor": {"id": 0, "firstBatch": []}},
-        op_type="must-use-primary",
-        not_master=not_master_reply,
-    ),
-]
-
+if _HAVE_MOCKUPDB:
+    not_master_reply = OpMsgReply(ok=0, errmsg="not master")
+
+    operations = [
+        Operation(
+            "find_one",
+            lambda client: client.db.collection.find_one(),
+            reply={"cursor": {"id": 0, "firstBatch": []}},
+            op_type="may-use-secondary",
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "count_documents",
+            lambda client: client.db.collection.count_documents({}),
+            reply={"n": 1},
+            op_type="may-use-secondary",
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "estimated_document_count",
+            lambda client: client.db.collection.estimated_document_count(),
+            reply={"n": 1},
+            op_type="may-use-secondary",
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "aggregate",
+            lambda client: client.db.collection.aggregate([]),
+            reply={"cursor": {"id": 0, "firstBatch": []}},
+            op_type="may-use-secondary",
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "options",
+            lambda client: client.db.collection.options(),
+            reply={"cursor": {"id": 0, "firstBatch": []}},
+            op_type="must-use-primary",
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "command",
+            lambda client: client.db.command("foo"),
+            reply={"ok": 1},
+            op_type="must-use-primary",  # Ignores client's read preference.
+            not_master=not_master_reply,
+        ),
+        Operation(
+            "secondary command",
+            lambda client: client.db.command("foo", read_preference=ReadPreference.SECONDARY),
+            reply={"ok": 1},
+            op_type="always-use-secondary",
+            not_master=OpReply(ok=0, errmsg="node is recovering"),
+        ),
+        Operation(
+            "listIndexes",
+            lambda client: client.db.collection.index_information(),
+            reply={"cursor": {"id": 0, "firstBatch": []}},
+            op_type="must-use-primary",
+            not_master=not_master_reply,
+        ),
+    ]
+else:
+    operations = []
 
 _ops_by_name = {op.name: op for op in operations}
 

test/mockupdb/test_auth_recovering_member.py

@@ -15,11 +15,20 @@
 
 import unittest
 
-from mockupdb import MockupDB
+import pytest
+
+try:
+    from mockupdb import MockupDB
+
+    _HAVE_MOCKUPDB = True
+except ImportError:
+    _HAVE_MOCKUPDB = False
 
 from pymongo import MongoClient
 from pymongo.errors import ServerSelectionTimeoutError
 
+pytestmark = pytest.mark.mockupdb
+
 
 class TestAuthRecoveringMember(unittest.TestCase):
     def test_auth_recovering_member(self):