Merge branch 'master' into PYTHON-3636

Author: ShaneHarvey

.evergreen/run-tests.sh

@@ -24,12 +24,6 @@ else
   exit 1
 fi
 
-# Source the local secrets export file if available.
-if [ -f "./secrets-export.sh" ]; then
-  echo "Sourcing local secrets file"
-  . "./secrets-export.sh"
-fi
-
 # List the packages.
 uv sync ${UV_ARGS} --reinstall
 uv pip list

.evergreen/scripts/run_server.py

@@ -28,11 +28,6 @@ def start_server():
     elif test_name == "load_balancer":
         set_env("LOAD_BALANCER")
 
-    elif test_name == "auth_oidc":
-        raise ValueError(
-            "OIDC auth does not use run-orchestration directly, do not use run-server!"
-        )
-
     elif test_name == "ocsp":
         opts.ssl = True
         if "ORCHESTRATION_FILE" not in os.environ:

.evergreen/scripts/setup_tests.py

@@ -115,8 +115,17 @@ def setup_libmongocrypt():
         run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
 
 
-def get_secrets(name: str) -> None:
-    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}")
+def load_config_from_file(path: str | Path) -> dict[str, str]:
+    config = read_env(path)
+    for key, value in config.items():
+        write_env(key, value)
+    return config
+
+
+def get_secrets(name: str) -> dict[str, str]:
+    secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling")
+    run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir)
+    return load_config_from_file(secrets_dir / "secrets-export.sh")
 
 
 def handle_test_env() -> None:
@@ -158,7 +167,7 @@ def handle_test_env() -> None:
 
     # Handle pass through env vars.
     for var in PASS_THROUGH_ENV:
-        if is_set(var):
+        if is_set(var) or getattr(opts, var.lower()):
             write_env(var, os.environ[var])
 
     if extra := EXTRAS_MAP.get(test_name, ""):
@@ -233,12 +242,11 @@ def handle_test_env() -> None:
     if is_set("MONGODB_URI"):
         write_env("PYMONGO_MUST_CONNECT", "true")
 
-    if is_set("DISABLE_TEST_COMMANDS"):
+    if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands:
         write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
 
     if test_name == "enterprise_auth":
-        get_secrets("drivers/enterprise_auth")
-        config = read_env(f"{ROOT}/secrets-export.sh")
+        config = get_secrets("drivers/enterprise_auth")
         if PLATFORM == "windows":
             LOGGER.info("Setting GSSAPI_PASS")
             write_env("GSSAPI_PASS", config["SASL_PASS"])
@@ -316,7 +324,7 @@ def handle_test_env() -> None:
         write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
         write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
 
-    compressors = os.environ.get("COMPRESSORS")
+    compressors = os.environ.get("COMPRESSORS") or opts.compressor
     if compressors == "snappy":
         UV_ARGS.append("--extra snappy")
     elif compressors == "zstd":
@@ -349,13 +357,15 @@ def handle_test_env() -> None:
     if test_name == "encryption":
         if not DRIVERS_TOOLS:
             raise RuntimeError("Missing DRIVERS_TOOLS")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh")
+        csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
+        run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir)
+        load_config_from_file(csfle_dir / "secrets-export.sh")
+        run_command(f"bash {csfle_dir}/start-servers.sh")
 
         if sub_test_name == "pyopenssl":
             UV_ARGS.append("--extra ocsp")
 
-    if is_set("TEST_CRYPT_SHARED"):
+    if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared:
         config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh")
         CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
         LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
@@ -414,15 +424,15 @@ def handle_test_env() -> None:
 
     # Add coverage if requested.
     # Only cover CPython. PyPy reports suspiciously low coverage.
-    if is_set("COVERAGE") and platform.python_implementation() == "CPython":
+    if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython":
         # Keep in sync with combine-coverage.sh.
         # coverage >=5 is needed for relative_files=true.
         UV_ARGS.append("--group coverage")
         TEST_ARGS = f"{TEST_ARGS} --cov"
         write_env("COVERAGE")
 
-    if is_set("GREEN_FRAMEWORK"):
-        framework = os.environ["GREEN_FRAMEWORK"]
+    if is_set("GREEN_FRAMEWORK") or opts.green_framework:
+        framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"]
         UV_ARGS.append(f"--group {framework}")
 
     else:

.evergreen/scripts/utils.py

@@ -52,7 +52,10 @@ class Distro:
 # Tests that require a sub test suite.
 SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
 
-EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"]
+EXTRA_TESTS = ["mod_wsgi", "aws_lambda"]
+
+# Tests that do not use run-orchestration.
+NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"]
 
 
 def get_test_options(
@@ -75,19 +78,47 @@ def get_test_options(
     else:
         parser.add_argument(
             "test_name",
-            choices=sorted(TEST_SUITE_MAP),
+            choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION),
             nargs="?",
             default="default",
             help="The optional name of the test suite to be run, which informs the server configuration.",
         )
     parser.add_argument(
-        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level"
+        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level."
     )
     parser.add_argument(
-        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level"
+        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level."
     )
-    parser.add_argument("--auth", action="store_true", help="Whether to add authentication")
-    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration")
+    parser.add_argument("--auth", action="store_true", help="Whether to add authentication.")
+    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.")
+
+    # Add the test modifiers.
+    if require_sub_test_name:
+        parser.add_argument(
+            "--debug-log", action="store_true", help="Enable pymongo standard logging."
+        )
+        parser.add_argument("--cov", action="store_true", help="Add test coverage.")
+        parser.add_argument(
+            "--green-framework",
+            nargs=1,
+            choices=["eventlet", "gevent"],
+            help="Optional green framework to test against.",
+        )
+        parser.add_argument(
+            "--compressor",
+            nargs=1,
+            choices=["zlib", "zstd", "snappy"],
+            help="Optional compression algorithm.",
+        )
+        parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.")
+        parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.")
+        parser.add_argument(
+            "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use."
+        )
+        parser.add_argument(
+            "--disable-test-commands", action="store_true", help="Disable test commands."
+        )
+
     # Get the options.
     if not allow_extra_opts:
         opts, extra_opts = parser.parse_args(), []
@@ -113,7 +144,7 @@ def get_test_options(
     return opts, extra_opts
 
 
-def read_env(path: Path | str) -> dict[str, Any]:
+def read_env(path: Path | str) -> dict[str, str]:
     config = dict()
     with Path(path).open() as fid:
         for line in fid.readlines():

CONTRIBUTING.md

@@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh.
 
 ### Usage
 
-- Run `just run-server` with optional args to set up the server.
-  All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`.
+- Run `just run-server` with optional args to set up the server.  All given options will be passed to
+  `run-orchestration.sh` in `$DRIVERS_TOOLS`.  See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h`
+  for a full list of options.
 - Run `just setup-tests` with optional args to set up the test environment, secrets, etc.
+  See `just setup-tests -h` for a full list of available options.
 - Run `just run-tests` to run the tests in an appropriate Python environment.
 - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server.
 
@@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step.
 - Run the tests: `just run-tests`.
 
 ## Enable Debug Logs
+
 - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`.
 - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine.
 - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`.
+- Finally, you can use `just setup-tests --debug-log`.
 - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch.
 
+## Adding a new test suite
+
+- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add
+  to the list of pytest markers in `pyproject.toml`.  Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`.  If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead.
+- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in
+  `.evergreen/scripts/utils.py`.
+- If there is something special required to run the local server or there is an extra flag that should always be set
+  like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`.
+- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`.  This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`.
+- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`.
+- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`.
+- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`.
+- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`.
+- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
+
 ## Re-sync Spec Tests
 
 If you would like to re-sync the copy of the specification tests in the

doc/changelog.rst

@@ -11,6 +11,9 @@ PyMongo 4.12 brings a number of changes including:
 - Support for $lookup in CSFLE and QE supported on MongoDB 8.1+.
 - AsyncMongoClient no longer performs DNS resolution for "mongodb+srv://" connection strings on creation.
   To avoid blocking the asyncio loop, the resolution is now deferred until the client is first connected.
+- Added index hinting support to the
+  :meth:`~pymongo.asynchronous.collection.AsyncCollection.distinct` and
+  :meth:`~pymongo.collection.Collection.distinct` commands.
 
 Issues Resolved
 ...............

pymongo/asynchronous/client_session.py

@@ -458,10 +458,10 @@ def _max_time_expired_error(exc: PyMongoError) -> bool:
 
 
 # From the transactions spec, all the retryable writes errors plus
-# WriteConcernFailed.
+# WriteConcernTimeout.
 _UNKNOWN_COMMIT_ERROR_CODES: frozenset = _RETRYABLE_ERROR_CODES | frozenset(
     [
-        64,  # WriteConcernFailed
+        64,  # WriteConcernTimeout
         50,  # MaxTimeMSExpired
     ]
 )

pymongo/asynchronous/collection.py

@@ -3111,6 +3111,7 @@ async def distinct(
         filter: Optional[Mapping[str, Any]] = None,
         session: Optional[AsyncClientSession] = None,
         comment: Optional[Any] = None,
+        hint: Optional[_IndexKeyHint] = None,
         **kwargs: Any,
     ) -> list:
         """Get a list of distinct values for `key` among all documents
@@ -3138,8 +3139,15 @@ async def distinct(
             :class:`~pymongo.asynchronous.client_session.AsyncClientSession`.
         :param comment: A user-provided comment to attach to this
             command.
+        :param hint: An index to use to support the query
+            predicate specified either by its string name, or in the same
+            format as passed to :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index`
+            (e.g. ``[('field', ASCENDING)]``).
         :param kwargs: See list of options above.
 
+        .. versionchanged:: 4.12
+           Added ``hint`` parameter.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
@@ -3158,6 +3166,10 @@ async def distinct(
         cmd.update(kwargs)
         if comment is not None:
             cmd["comment"] = comment
+        if hint is not None:
+            if not isinstance(hint, str):
+                hint = helpers_shared._index_document(hint)
+            cmd["hint"] = hint  # type: ignore[assignment]
 
         async def _cmd(
             session: Optional[AsyncClientSession],

pymongo/synchronous/client_session.py

@@ -457,10 +457,10 @@ def _max_time_expired_error(exc: PyMongoError) -> bool:
 
 
 # From the transactions spec, all the retryable writes errors plus
-# WriteConcernFailed.
+# WriteConcernTimeout.
 _UNKNOWN_COMMIT_ERROR_CODES: frozenset = _RETRYABLE_ERROR_CODES | frozenset(
     [
-        64,  # WriteConcernFailed
+        64,  # WriteConcernTimeout
         50,  # MaxTimeMSExpired
     ]
 )

pymongo/synchronous/collection.py

@@ -3104,6 +3104,7 @@ def distinct(
         filter: Optional[Mapping[str, Any]] = None,
         session: Optional[ClientSession] = None,
         comment: Optional[Any] = None,
+        hint: Optional[_IndexKeyHint] = None,
         **kwargs: Any,
     ) -> list:
         """Get a list of distinct values for `key` among all documents
@@ -3131,8 +3132,15 @@ def distinct(
             :class:`~pymongo.client_session.ClientSession`.
         :param comment: A user-provided comment to attach to this
             command.
+        :param hint: An index to use to support the query
+            predicate specified either by its string name, or in the same
+            format as passed to :meth:`~pymongo.collection.Collection.create_index`
+            (e.g. ``[('field', ASCENDING)]``).
         :param kwargs: See list of options above.
 
+        .. versionchanged:: 4.12
+           Added ``hint`` parameter.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 
@@ -3151,6 +3159,10 @@ def distinct(
         cmd.update(kwargs)
         if comment is not None:
             cmd["comment"] = comment
+        if hint is not None:
+            if not isinstance(hint, str):
+                hint = helpers_shared._index_document(hint)
+            cmd["hint"] = hint  # type: ignore[assignment]
 
         def _cmd(
             session: Optional[ClientSession],

test/asynchronous/unified_format.py

@@ -568,7 +568,11 @@ def maybe_skip_test(self, spec):
                 self.skipTest("CSOT not implemented for watch()")
             if "cursors" in class_name:
                 self.skipTest("CSOT not implemented for cursors")
-            if "tailable" in class_name:
+            if (
+                "tailable" in class_name
+                or "tailable" in description
+                and "non-tailable" not in description
+            ):
                 self.skipTest("CSOT not implemented for tailable cursors")
             if "sessions" in class_name:
                 self.skipTest("CSOT not implemented for sessions")

test/bson_binary_vector/float32.json

@@ -32,7 +32,7 @@
     {
       "description": "Infinity Vector FLOAT32",
       "valid": true,
-      "vector": ["-inf", 0.0, "inf"],
+      "vector": [{"$numberDouble": "-Infinity"}, 0.0, {"$numberDouble": "Infinity"} ],
       "dtype_hex": "0x27",
       "dtype_alias": "FLOAT32",
       "padding": 0,