Merge branch 'master' into PYTHON-5109

Author: NoahStapp

test/asynchronous/helpers.py

@@ -15,6 +15,7 @@
 """Shared constants and helper methods for pymongo, bson, and gridfs test suites."""
 from __future__ import annotations
 
+import asyncio
 import base64
 import gc
 import multiprocessing
@@ -30,6 +31,8 @@
 import warnings
 from asyncio import iscoroutinefunction
 
+from pymongo._asyncio_task import create_task
+
 try:
     import ipaddress
 
@@ -369,3 +372,37 @@ def disable(self):
             os.environ.pop("SSL_CERT_FILE")
         else:
             os.environ["SSL_CERT_FILE"] = self.original_certs
+
+
+if _IS_SYNC:
+    PARENT = threading.Thread
+else:
+    PARENT = object
+
+
+class ConcurrentRunner(PARENT):
+    def __init__(self, name, *args, **kwargs):
+        if _IS_SYNC:
+            super().__init__(*args, **kwargs)
+        self.name = name
+        self.stopped = False
+        self.task = None
+        if "target" in kwargs:
+            self.target = kwargs["target"]
+
+    if not _IS_SYNC:
+
+        async def start(self):
+            self.task = create_task(self.run(), name=self.name)
+
+        async def join(self, timeout: float | None = 0):  # type: ignore[override]
+            if self.task is not None:
+                await asyncio.wait([self.task], timeout=timeout)
+
+        def is_alive(self):
+            return not self.stopped
+
+    async def run(self):
+        if self.target:
+            await self.target()
+        self.stopped = True

test/asynchronous/test_run_command.py

@@ -0,0 +1,41 @@
+# Copyright 2024-Present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run Command unified tests."""
+from __future__ import annotations
+
+import os
+import unittest
+from pathlib import Path
+from test.asynchronous.unified_format import generate_test_classes
+
+_IS_SYNC = False
+
+# Location of JSON test specifications.
+if _IS_SYNC:
+    TEST_PATH = os.path.join(Path(__file__).resolve().parent, "run_command")
+else:
+    TEST_PATH = os.path.join(Path(__file__).resolve().parent.parent, "run_command")
+
+
+globals().update(
+    generate_test_classes(
+        os.path.join(TEST_PATH, "unified"),
+        module=__name__,
+    )
+)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/asynchronous/test_unified_format.py

@@ -0,0 +1,99 @@
+# Copyright 2020-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+sys.path[0:0] = [""]
+
+from test import UnitTest, unittest
+from test.asynchronous.unified_format import MatchEvaluatorUtil, generate_test_classes
+
+from bson import ObjectId
+
+_IS_SYNC = False
+
+# Location of JSON test specifications.
+if _IS_SYNC:
+    TEST_PATH = os.path.join(Path(__file__).resolve().parent, "unified-test-format")
+else:
+    TEST_PATH = os.path.join(Path(__file__).resolve().parent.parent, "unified-test-format")
+
+
+globals().update(
+    generate_test_classes(
+        os.path.join(TEST_PATH, "valid-pass"),
+        module=__name__,
+        class_name_prefix="UnifiedTestFormat",
+        expected_failures=[
+            "Client side error in command starting transaction",  # PYTHON-1894
+        ],
+        RUN_ON_SERVERLESS=False,
+    )
+)
+
+
+globals().update(
+    generate_test_classes(
+        os.path.join(TEST_PATH, "valid-fail"),
+        module=__name__,
+        class_name_prefix="UnifiedTestFormat",
+        bypass_test_generation_errors=True,
+        expected_failures=[
+            ".*",  # All tests expected to fail
+        ],
+        RUN_ON_SERVERLESS=False,
+    )
+)
+
+
+class TestMatchEvaluatorUtil(UnitTest):
+    def setUp(self):
+        self.match_evaluator = MatchEvaluatorUtil(self)
+
+    def test_unsetOrMatches(self):
+        spec: dict[str, Any] = {"$$unsetOrMatches": {"y": {"$$unsetOrMatches": 2}}}
+        for actual in [{}, {"y": 2}, None]:
+            self.match_evaluator.match_result(spec, actual)
+
+        spec = {"x": {"$$unsetOrMatches": {"y": {"$$unsetOrMatches": 2}}}}
+        for actual in [{}, {"x": {}}, {"x": {"y": 2}}]:
+            self.match_evaluator.match_result(spec, actual)
+
+        spec = {"y": {"$$unsetOrMatches": {"$$exists": True}}}
+        self.match_evaluator.match_result(spec, {})
+        self.match_evaluator.match_result(spec, {"y": 2})
+        self.match_evaluator.match_result(spec, {"x": 1})
+        self.match_evaluator.match_result(spec, {"y": {}})
+
+    def test_type(self):
+        self.match_evaluator.match_result(
+            {
+                "operationType": "insert",
+                "ns": {"db": "change-stream-tests", "coll": "test"},
+                "fullDocument": {"_id": {"$$type": "objectId"}, "x": 1},
+            },
+            {
+                "operationType": "insert",
+                "fullDocument": {"_id": ObjectId("5fc93511ac93941052098f0c"), "x": 1},
+                "ns": {"db": "change-stream-tests", "coll": "test"},
+            },
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/asynchronous/unified_format.py

@@ -35,6 +35,7 @@
     client_knobs,
     unittest,
 )
+from test.asynchronous.utils_spec_runner import SpecRunnerTask
 from test.unified_format_shared import (
     KMS_TLS_OPTS,
     PLACEHOLDER_MAP,
@@ -58,7 +59,6 @@
     snake_to_camel,
     wait_until,
 )
-from test.utils_spec_runner import SpecRunnerThread
 from test.version import Version
 from typing import Any, Dict, List, Mapping, Optional
 
@@ -382,8 +382,8 @@ async def drop(self: AsyncGridFSBucket, *args: Any, **kwargs: Any) -> None:
             return
         elif entity_type == "thread":
             name = spec["id"]
-            thread = SpecRunnerThread(name)
-            thread.start()
+            thread = SpecRunnerTask(name)
+            await thread.start()
             self[name] = thread
             return
 
@@ -711,7 +711,7 @@ async def _databaseOperation_runCommand(self, target, **kwargs):
         return await target.command(**kwargs)
 
     async def _databaseOperation_runCursorCommand(self, target, **kwargs):
-        return list(await self._databaseOperation_createCommandCursor(target, **kwargs))
+        return await (await self._databaseOperation_createCommandCursor(target, **kwargs)).to_list()
 
     async def _databaseOperation_createCommandCursor(self, target, **kwargs):
         self.__raise_if_unsupported("createCommandCursor", target, AsyncDatabase)
@@ -1177,16 +1177,16 @@ def primary_changed() -> bool:
 
         wait_until(primary_changed, "change primary", timeout=timeout)
 
-    def _testOperation_runOnThread(self, spec):
+    async def _testOperation_runOnThread(self, spec):
         """Run the 'runOnThread' operation."""
         thread = self.entity_map[spec["thread"]]
-        thread.schedule(lambda: self.run_entity_operation(spec["operation"]))
+        await thread.schedule(functools.partial(self.run_entity_operation, spec["operation"]))
 
-    def _testOperation_waitForThread(self, spec):
+    async def _testOperation_waitForThread(self, spec):
         """Run the 'waitForThread' operation."""
         thread = self.entity_map[spec["thread"]]
-        thread.stop()
-        thread.join(10)
+        await thread.stop()
+        await thread.join(10)
         if thread.exc:
             raise thread.exc
         self.assertFalse(thread.is_alive(), "Thread {} is still running".format(spec["thread"]))

test/asynchronous/utils_spec_runner.py

@@ -18,11 +18,11 @@
 import asyncio
 import functools
 import os
-import threading
 import unittest
 from asyncio import iscoroutinefunction
 from collections import abc
 from test.asynchronous import AsyncIntegrationTest, async_client_context, client_knobs
+from test.asynchronous.helpers import ConcurrentRunner
 from test.utils import (
     CMAPListener,
     CompareType,
@@ -47,6 +47,7 @@
 from pymongo.asynchronous.command_cursor import AsyncCommandCursor
 from pymongo.asynchronous.cursor import AsyncCursor
 from pymongo.errors import AutoReconnect, BulkWriteError, OperationFailure, PyMongoError
+from pymongo.lock import _async_cond_wait, _async_create_condition, _async_create_lock
 from pymongo.read_concern import ReadConcern
 from pymongo.read_preferences import ReadPreference
 from pymongo.results import BulkWriteResult, _WriteResult
@@ -55,38 +56,36 @@
 _IS_SYNC = False
 
 
-class SpecRunnerThread(threading.Thread):
+class SpecRunnerTask(ConcurrentRunner):
     def __init__(self, name):
-        super().__init__()
-        self.name = name
+        super().__init__(name)
         self.exc = None
         self.daemon = True
-        self.cond = threading.Condition()
+        self.cond = _async_create_condition(_async_create_lock())
         self.ops = []
-        self.stopped = False
 
-    def schedule(self, work):
+    async def schedule(self, work):
         self.ops.append(work)
-        with self.cond:
+        async with self.cond:
             self.cond.notify()
 
-    def stop(self):
+    async def stop(self):
         self.stopped = True
-        with self.cond:
+        async with self.cond:
             self.cond.notify()
 
-    def run(self):
+    async def run(self):
         while not self.stopped or self.ops:
             if not self.ops:
-                with self.cond:
-                    self.cond.wait(10)
+                async with self.cond:
+                    await _async_cond_wait(self.cond, 10)
             if self.ops:
                 try:
                     work = self.ops.pop(0)
-                    work()
+                    await work()
                 except Exception as exc:
                     self.exc = exc
-                    self.stop()
+                    await self.stop()
 
 
 class AsyncSpecTestCreator:

test/helpers.py

@@ -15,6 +15,7 @@
 """Shared constants and helper methods for pymongo, bson, and gridfs test suites."""
 from __future__ import annotations
 
+import asyncio
 import base64
 import gc
 import multiprocessing
@@ -30,6 +31,8 @@
 import warnings
 from asyncio import iscoroutinefunction
 
+from pymongo._asyncio_task import create_task
+
 try:
     import ipaddress
 
@@ -369,3 +372,37 @@ def disable(self):
             os.environ.pop("SSL_CERT_FILE")
         else:
             os.environ["SSL_CERT_FILE"] = self.original_certs
+
+
+if _IS_SYNC:
+    PARENT = threading.Thread
+else:
+    PARENT = object
+
+
+class ConcurrentRunner(PARENT):
+    def __init__(self, name, *args, **kwargs):
+        if _IS_SYNC:
+            super().__init__(*args, **kwargs)
+        self.name = name
+        self.stopped = False
+        self.task = None
+        if "target" in kwargs:
+            self.target = kwargs["target"]
+
+    if not _IS_SYNC:
+
+        def start(self):
+            self.task = create_task(self.run(), name=self.name)
+
+        def join(self, timeout: float | None = 0):  # type: ignore[override]
+            if self.task is not None:
+                asyncio.wait([self.task], timeout=timeout)
+
+        def is_alive(self):
+            return not self.stopped
+
+    def run(self):
+        if self.target:
+            self.target()
+        self.stopped = True