Merge branch 'master' into PYTHON-5081

Author: sleepyStick

.evergreen/generated_configs/tasks.yml

@@ -817,97 +817,110 @@ buildvariants:
       PYTHON_BINARY: /opt/python/3.13/bin/python3
 
   # Ocsp tests
-  - name: ocsp-rhel8-v4.4-python3.9
+  - name: ocsp-rhel8-v4.2-python3.9
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v4.4 Python3.9
+    display_name: OCSP RHEL8 v4.2 Python3.9
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: "4.4"
+      VERSION: "4.2"
       PYTHON_BINARY: /opt/python/3.9/bin/python3
-  - name: ocsp-rhel8-v5.0-python3.10
+  - name: ocsp-rhel8-v4.4-python3.10
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v5.0 Python3.10
+    display_name: OCSP RHEL8 v4.4 Python3.10
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: "5.0"
+      VERSION: "4.4"
       PYTHON_BINARY: /opt/python/3.10/bin/python3
-  - name: ocsp-rhel8-v6.0-python3.11
+  - name: ocsp-rhel8-v5.0-python3.11
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v6.0 Python3.11
+    display_name: OCSP RHEL8 v5.0 Python3.11
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: "6.0"
+      VERSION: "5.0"
       PYTHON_BINARY: /opt/python/3.11/bin/python3
-  - name: ocsp-rhel8-v7.0-python3.12
+  - name: ocsp-rhel8-v6.0-python3.12
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v7.0 Python3.12
+    display_name: OCSP RHEL8 v6.0 Python3.12
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: "7.0"
+      VERSION: "6.0"
       PYTHON_BINARY: /opt/python/3.12/bin/python3
-  - name: ocsp-rhel8-v8.0-python3.13
+  - name: ocsp-rhel8-v7.0-python3.13
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 v8.0 Python3.13
+    display_name: OCSP RHEL8 v7.0 Python3.13
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: "8.0"
+      VERSION: "7.0"
       PYTHON_BINARY: /opt/python/3.13/bin/python3
-  - name: ocsp-rhel8-rapid-pypy3.10
+  - name: ocsp-rhel8-v8.0-pypy3.10
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 rapid PyPy3.10
+    display_name: OCSP RHEL8 v8.0 PyPy3.10
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: rapid
+      VERSION: "8.0"
       PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
-  - name: ocsp-rhel8-latest-python3.9
+  - name: ocsp-rhel8-rapid-python3.9
     tasks:
       - name: .ocsp
-    display_name: OCSP RHEL8 latest Python3.9
+    display_name: OCSP RHEL8 rapid Python3.9
     run_on:
       - rhel87-small
     batchtime: 20160
     expansions:
       AUTH: noauth
       SSL: ssl
       TOPOLOGY: server
-      VERSION: latest
+      VERSION: rapid
       PYTHON_BINARY: /opt/python/3.9/bin/python3
+  - name: ocsp-rhel8-latest-python3.10
+    tasks:
+      - name: .ocsp
+    display_name: OCSP RHEL8 latest Python3.10
+    run_on:
+      - rhel87-small
+    batchtime: 20160
+    expansions:
+      AUTH: noauth
+      SSL: ssl
+      TOPOLOGY: server
+      VERSION: latest
+      PYTHON_BINARY: /opt/python/3.10/bin/python3
   - name: ocsp-win64-v4.4-python3.9
     tasks:
       - name: .ocsp-rsa !.ocsp-staple
@@ -1338,6 +1351,7 @@ buildvariants:
   - name: storage-inmemory-rhel8-python3.9
     tasks:
       - name: .standalone .noauth .nossl .4.0 .sync_async
+      - name: .standalone .noauth .nossl .4.2 .sync_async
       - name: .standalone .noauth .nossl .4.4 .sync_async
       - name: .standalone .noauth .nossl .5.0 .sync_async
       - name: .standalone .noauth .nossl .6.0 .sync_async

.evergreen/generated_configs/variants.yml

@@ -26,7 +26,7 @@
 # Globals
 ##############
 
-ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"]
+ALL_VERSIONS = ["4.0", "4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"]
 CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"]
 PYPYS = ["pypy3.10"]
 ALL_PYTHONS = CPYTHONS + PYPYS

.evergreen/scripts/generate_config.py

@@ -35,10 +35,9 @@ jobs:
         # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
         buildplat:
         - [ubuntu-20.04, "manylinux_x86_64", "cp3*-manylinux_x86_64"]
-        - [ubuntu-24.04-arm, "manylinux_aarch64", "cp3*-manylinux_aarch64"]
-        # Disabled pending PYTHON-5058
-        # - [ubuntu-24.04, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"]
-        # - [ubuntu-24.04, "manylinux_s390x", "cp3*-manylinux_s390x"]
+        - [ubuntu-20.04, "manylinux_aarch64", "cp3*-manylinux_aarch64"]
+        - [ubuntu-20.04, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"]
+        - [ubuntu-20.04, "manylinux_s390x", "cp3*-manylinux_s390x"]
         - [ubuntu-20.04, "manylinux_i686", "cp3*-manylinux_i686"]
         - [windows-2019, "win_amd6", "cp3*-win_amd64"]
         - [windows-2019, "win32", "cp3*-win32"]
@@ -63,6 +62,10 @@ jobs:
         if: runner.os == 'Linux'
         uses: docker/setup-qemu-action@v3
         with:
+          # setup-qemu-action by default uses `tonistiigi/binfmt:latest` image,
+          # which is out of date. This causes seg faults during build.
+          # Here we manually fix the version.
+          image: tonistiigi/binfmt:qemu-v8.1.5
           platforms: all
 
       - name: Install cibuildwheel

.github/workflows/dist.yml

@@ -37,6 +37,7 @@ jobs:
   pre-publish:
     environment: release
     runs-on: ubuntu-latest
+    if: github.repository_owner == 'mongodb' || github.event_name == 'workflow_dispatch'
     permissions:
       id-token: write
       contents: write

.github/workflows/release-python.yml

@@ -261,6 +261,11 @@ To prevent the `synchro` hook from accidentally overwriting code, it first check
 of a file is changing and not its async counterpart, and will fail.
 In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`.
 
+Sometimes, the `synchro` hook will fail and introduce changes many previously unmodified files. This is due to static
+Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues,
+run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the `synchro`
+hook again.
+
 ## Converting a test to async
 The `tools/convert_test_to_async.py` script takes in an existing synchronous test file and outputs a
 partially-converted asynchronous version of the same name to the `test/asynchronous` directory.

CONTRIBUTING.md

@@ -0,0 +1,199 @@
+# Copyright 2015-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test AsyncMongoClient's mongos load balancing using a mock."""
+from __future__ import annotations
+
+import asyncio
+import sys
+import threading
+from test.asynchronous.helpers import ConcurrentRunner
+
+from pymongo.operations import _Op
+
+sys.path[0:0] = [""]
+
+from test.asynchronous import AsyncMockClientTest, async_client_context, connected, unittest
+from test.asynchronous.pymongo_mocks import AsyncMockClient
+from test.utils import async_wait_until
+
+from pymongo.errors import AutoReconnect, InvalidOperation
+from pymongo.server_selectors import writable_server_selector
+from pymongo.topology_description import TOPOLOGY_TYPE
+
+_IS_SYNC = False
+
+
+class SimpleOp(ConcurrentRunner):
+    def __init__(self, client):
+        super().__init__()
+        self.client = client
+        self.passed = False
+
+    async def run(self):
+        await self.client.db.command("ping")
+        self.passed = True  # No exception raised.
+
+
+async def do_simple_op(client, ntasks):
+    tasks = [SimpleOp(client) for _ in range(ntasks)]
+    for t in tasks:
+        await t.start()
+
+    for t in tasks:
+        await t.join()
+
+    for t in tasks:
+        assert t.passed
+
+
+async def writable_addresses(topology):
+    return {
+        server.description.address
+        for server in await topology.select_servers(writable_server_selector, _Op.TEST)
+    }
+
+
+class TestMongosLoadBalancing(AsyncMockClientTest):
+    @async_client_context.require_connection
+    @async_client_context.require_no_load_balancer
+    async def asyncSetUp(self):
+        await super().asyncSetUp()
+
+    def mock_client(self, **kwargs):
+        mock_client = AsyncMockClient(
+            standalones=[],
+            members=[],
+            mongoses=["a:1", "b:2", "c:3"],
+            host="a:1,b:2,c:3",
+            connect=False,
+            **kwargs,
+        )
+        self.addAsyncCleanup(mock_client.aclose)
+
+        # Latencies in seconds.
+        mock_client.mock_rtts["a:1"] = 0.020
+        mock_client.mock_rtts["b:2"] = 0.025
+        mock_client.mock_rtts["c:3"] = 0.045
+        return mock_client
+
+    async def test_lazy_connect(self):
+        # While connected() ensures we can trigger connection from the main
+        # thread and wait for the monitors, this test triggers connection from
+        # several threads at once to check for data races.
+        nthreads = 10
+        client = self.mock_client()
+        self.assertEqual(0, len(client.nodes))
+
+        # Trigger initial connection.
+        await do_simple_op(client, nthreads)
+        await async_wait_until(lambda: len(client.nodes) == 3, "connect to all mongoses")
+
+    async def test_failover(self):
+        ntasks = 10
+        client = await connected(self.mock_client(localThresholdMS=0.001))
+        await async_wait_until(lambda: len(client.nodes) == 3, "connect to all mongoses")
+
+        # Our chosen mongos goes down.
+        client.kill_host("a:1")
+
+        # Trigger failover to higher-latency nodes. AutoReconnect should be
+        # raised at most once in each thread.
+        passed = []
+
+        async def f():
+            try:
+                await client.db.command("ping")
+            except AutoReconnect:
+                # Second attempt succeeds.
+                await client.db.command("ping")
+
+            passed.append(True)
+
+        tasks = [ConcurrentRunner(target=f) for _ in range(ntasks)]
+        for t in tasks:
+            await t.start()
+
+        for t in tasks:
+            await t.join()
+
+        self.assertEqual(ntasks, len(passed))
+
+        # Down host removed from list.
+        self.assertEqual(2, len(client.nodes))
+
+    async def test_local_threshold(self):
+        client = await connected(self.mock_client(localThresholdMS=30))
+        self.assertEqual(30, client.options.local_threshold_ms)
+        await async_wait_until(lambda: len(client.nodes) == 3, "connect to all mongoses")
+        topology = client._topology
+
+        # All are within a 30-ms latency window, see self.mock_client().
+        self.assertEqual({("a", 1), ("b", 2), ("c", 3)}, await writable_addresses(topology))
+
+        # No error
+        await client.admin.command("ping")
+
+        client = await connected(self.mock_client(localThresholdMS=0))
+        self.assertEqual(0, client.options.local_threshold_ms)
+        # No error
+        await client.db.command("ping")
+        # Our chosen mongos goes down.
+        client.kill_host("{}:{}".format(*next(iter(client.nodes))))
+        try:
+            await client.db.command("ping")
+        except:
+            pass
+
+        # We eventually connect to a new mongos.
+        async def connect_to_new_mongos():
+            try:
+                return await client.db.command("ping")
+            except AutoReconnect:
+                pass
+
+        await async_wait_until(connect_to_new_mongos, "connect to a new mongos")
+
+    async def test_load_balancing(self):
+        # Although the server selection JSON tests already prove that
+        # select_servers works for sharded topologies, here we do an end-to-end
+        # test of discovering servers' round trip times and configuring
+        # localThresholdMS.
+        client = await connected(self.mock_client())
+        await async_wait_until(lambda: len(client.nodes) == 3, "connect to all mongoses")
+
+        # Prohibited for topology type Sharded.
+        with self.assertRaises(InvalidOperation):
+            await client.address
+
+        topology = client._topology
+        self.assertEqual(TOPOLOGY_TYPE.Sharded, topology.description.topology_type)
+
+        # a and b are within the 15-ms latency window, see self.mock_client().
+        self.assertEqual({("a", 1), ("b", 2)}, await writable_addresses(topology))
+
+        client.mock_rtts["a:1"] = 0.045
+
+        # Discover only b is within latency window.
+        async def predicate():
+            return {("b", 2)} == await writable_addresses(topology)
+
+        await async_wait_until(
+            predicate,
+            'discover server "a" is too far',
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()