PYTHON-4976 Replace hatch with uv as our python environment and workfow tool (#2068)

Author: blink1073

.evergreen/run-tests.sh

@@ -37,8 +37,7 @@ export PIP_QUIET=1  # Quiet by default
 export PIP_PREFER_BINARY=1 # Prefer binary dists by default
 
 set +x
-python -c "import sys; sys.exit(sys.prefix == sys.base_prefix)" || (echo "Not inside a virtual env!"; exit 1)
-PYTHON_IMPL=$(python -c "import platform; print(platform.python_implementation())")
+PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())")
 
 # Try to source local Drivers Secrets
 if [ -f ./secrets-export.sh ]; then
@@ -48,9 +47,13 @@ else
   echo "Not sourcing secrets"
 fi
 
-# Ensure C extensions have compiled.
+# Start compiling the args we'll pass to uv.
+# Run in an isolated environment so as not to pollute the base venv.
+UV_ARGS=("--isolated --extra test")
+
+# Ensure C extensions if applicable.
 if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
-    python tools/fail_if_no_c.py
+    uv run tools/fail_if_no_c.py
 fi
 
 if [ "$AUTH" != "noauth" ]; then
@@ -77,7 +80,7 @@ if [ "$AUTH" != "noauth" ]; then
 fi
 
 if [ -n "$TEST_ENTERPRISE_AUTH" ]; then
-    python -m pip install '.[gssapi]'
+    UV_ARGS+=("--extra gssapi")
     if [ "Windows_NT" = "$OS" ]; then
         echo "Setting GSSAPI_PASS"
         export GSSAPI_PASS=${SASL_PASS}
@@ -118,24 +121,26 @@ if [ "$SSL" != "nossl" ]; then
 fi
 
 if [ "$COMPRESSORS" = "snappy" ]; then
-    python -m pip install '.[snappy]'
+    UV_ARGS+=("--extra snappy")
 elif [ "$COMPRESSORS" = "zstd" ]; then
-    python -m pip install zstandard
+    UV_ARGS+=("--extra zstandard")
 fi
 
 # PyOpenSSL test setup.
 if [ -n "$TEST_PYOPENSSL" ]; then
-    python -m pip install '.[ocsp]'
+    UV_ARGS+=("--extra ocsp")
 fi
 
 if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
-    # Check for libmongocrypt checkout.
+    # Check for libmongocrypt download.
     if [ ! -d "libmongocrypt" ]; then
         echo "Run encryption setup first!"
         exit 1
     fi
 
-    python -m pip install '.[encryption]'
+    UV_ARGS+=("--extra encryption")
+    # TODO: Test with 'pip install pymongocrypt'
+    UV_ARGS+=("--group pymongocrypt_source")
 
     # Use the nocrypto build to avoid dependency issues with older windows/python versions.
     BASE=$(pwd)/libmongocrypt/nocrypto
@@ -155,21 +160,17 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
         exit 1
     fi
     export PYMONGOCRYPT_LIB
-
-    # TODO: Test with 'pip install pymongocrypt'
-    if [ ! -d "libmongocrypt_git" ]; then
-      git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    fi
-    python -m pip install -U setuptools
-    python -m pip install ./libmongocrypt_git/bindings/python
-    python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
-    python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
-    # PATH is updated by PREPARE_SHELL for access to mongocryptd.
+    # Ensure pymongocrypt is working properly.
+    # shellcheck disable=SC2048
+    uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
+    # shellcheck disable=SC2048
+    uv run ${UV_ARGS[*]} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
+    # PATH is updated by configure-env.sh for access to mongocryptd.
 fi
 
 if [ -n "$TEST_ENCRYPTION" ]; then
     if [ -n "$TEST_ENCRYPTION_PYOPENSSL" ]; then
-        python -m pip install '.[ocsp]'
+        UV_ARGS+=("--extra ocsp")
     fi
 
     if [ -n "$TEST_CRYPT_SHARED" ]; then
@@ -214,31 +215,31 @@ if [ -n "$TEST_ATLAS" ]; then
 fi
 
 if [ -n "$TEST_OCSP" ]; then
-    python -m pip install ".[ocsp]"
+    UV_ARGS+=("--extra ocsp")
     TEST_SUITES="ocsp"
 fi
 
 if [ -n "$TEST_AUTH_AWS" ]; then
-    python -m pip install ".[aws]"
+    UV_ARGS+=("--extra aws")
     TEST_SUITES="auth_aws"
 fi
 
 if [ -n "$TEST_AUTH_OIDC" ]; then
-    python -m pip install ".[aws]"
+    UV_ARGS+=("--extra aws")
     TEST_SUITES="auth_oidc"
 fi
 
 if [ -n "$PERF_TEST" ]; then
-    python -m pip install simplejson
+    UV_ARGS+=("--group perf")
     start_time=$(date +%s)
     TEST_SUITES="perf"
     # PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
     # affects the benchmark results.
     TEST_ARGS="test/performance/perf_test.py $TEST_ARGS"
 fi
 
-echo "Running $AUTH tests over $SSL with python $(which python)"
-python -c 'import sys; print(sys.version)'
+echo "Running $AUTH tests over $SSL with python $(uv python find)"
+uv run python -c 'import sys; print(sys.version)'
 
 
 # Run the tests, and store the results in Evergreen compatible XUnit XML
@@ -249,27 +250,30 @@ python -c 'import sys; print(sys.version)'
 if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then
     # Keep in sync with combine-coverage.sh.
     # coverage >=5 is needed for relative_files=true.
-    python -m pip install pytest-cov "coverage>=5,<=7.5"
+    UV_ARGS+=("--group coverage")
     TEST_ARGS="$TEST_ARGS --cov"
 fi
 
 if [ -n "$GREEN_FRAMEWORK" ]; then
-    python -m pip install $GREEN_FRAMEWORK
+    UV_ARGS+=("--group $GREEN_FRAMEWORK")
 fi
 
 # Show the installed packages
-PIP_QUIET=0 python -m pip list
+# shellcheck disable=SC2048
+PIP_QUIET=0 uv run ${UV_ARGS[*]} --with pip pip list
 
 if [ -z "$GREEN_FRAMEWORK" ]; then
     # Use --capture=tee-sys so pytest prints test output inline:
     # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
-    if [ -z "$TEST_SUITES" ]; then
-      python -m pytest -v --capture=tee-sys --durations=5 $TEST_ARGS
-    else
-      python -m pytest -v --capture=tee-sys --durations=5 -m $TEST_SUITES $TEST_ARGS
+    PYTEST_ARGS="-v --capture=tee-sys --durations=5 $TEST_ARGS"
+    if [ -n "$TEST_SUITES" ]; then
+      PYTEST_ARGS="-m $TEST_SUITES $PYTEST_ARGS"
     fi
+    # shellcheck disable=SC2048
+    uv run ${UV_ARGS[*]} pytest $PYTEST_ARGS
 else
-    python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
+    # shellcheck disable=SC2048
+    uv run ${UV_ARGS[*]} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
 fi
 
 # Handle perf test post actions.

.evergreen/scripts/configure-env.sh

@@ -14,14 +14,16 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
-HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml
+UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
+UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
-    HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
+    UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
+    UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
 fi
 
 SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@@ -62,7 +64,9 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
 
 export CARGO_HOME="$CARGO_HOME"
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-export HATCH_CONFIG="$HATCH_CONFIG"
+export UV_TOOL_DIR="$UV_TOOL_DIR"
+export UV_CACHE_DIR="$UV_CACHE_DIR"
+export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
 export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH"
 # shellcheck disable=SC2154
 export PROJECT="${project:-mongo-python-driver}"

.evergreen/scripts/generate_config.py

@@ -6,7 +6,7 @@
 # ]
 # ///
 
-# Note: Run this file with `hatch run`, `pipx run`, or `uv run`.
+# Note: Run this file with `pipx run`, or `uv run`.
 from __future__ import annotations
 
 import sys

.evergreen/scripts/install-dependencies.sh

@@ -40,3 +40,16 @@ if ! command -v just 2>/dev/null; then
   fi
   echo "Installing just... done."
 fi
+
+# Install uv.
+if ! command -v uv 2>/dev/null; then
+  echo "Installing uv..."
+  # On most systems we can install directly.
+  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || {
+     _pip_install uv uv
+  }
+  if ! command -v uv 2>/dev/null; then
+    export PATH="$PATH:$_BIN_DIR"
+  fi
+  echo "Installing uv... done."
+fi

.evergreen/scripts/setup-dev-env.sh

@@ -6,69 +6,37 @@ HERE=$(dirname ${BASH_SOURCE:-$0})
 pushd "$(dirname "$(dirname $HERE)")" > /dev/null
 
 # Source the env file to pick up common variables.
-if [ -f $HERE/scripts/env.sh ]; then
-  source $HERE/scripts/env.sh
+if [ -f $HERE/env.sh ]; then
+  source $HERE/env.sh
 fi
 
+# Ensure dependencies are installed.
+. $HERE/install-dependencies.sh
+
+
 # Set the location of the python bin dir.
 if [ "Windows_NT" = "${OS:-}" ]; then
   BIN_DIR=.venv/Scripts
 else
   BIN_DIR=.venv/bin
 fi
 
-. $HERE/install-dependencies.sh
-
 # Ensure there is a python venv.
 if [ ! -d $BIN_DIR ]; then
   . .evergreen/utils.sh
 
   if [ -z "${PYTHON_BINARY:-}" ]; then
       PYTHON_BINARY=$(find_python3)
   fi
-
-  echo "Creating virtual environment..."
-  createvirtualenv "$PYTHON_BINARY" .venv
-  echo "Creating virtual environment... done."
-fi
-
-# Activate the virtual env.
-. $BIN_DIR/activate
-
-# Ensure there is a local hatch.
-if [ ! -f $BIN_DIR/hatch ]; then
-  echo "Installing hatch..."
-  python -m pip install hatch || {
-    # CARGO_HOME is defined in configure-env.sh
-    export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/}
-    export RUSTUP_HOME="${CARGO_HOME}/.rustup"
-    ${DRIVERS_TOOLS}/.evergreen/install-rust.sh
-    source "${CARGO_HOME}/env"
-    python -m pip install hatch
-  }
-  echo "Installing hatch... done."
-fi
-
-# Ensure hatch does not write to user or global locations.
-HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml}
-if [ ! -f ${HATCH_CONFIG} ]; then
-  touch hatch_config.toml
-  hatch config restore
-  hatch config set dirs.data "$(pwd)/.hatch/data"
-  hatch config set dirs.cache "$(pwd)/.hatch/cache"
+  export UV_PYTHON=${PYTHON_BINARY}
+  echo "export UV_PYTHON=$UV_PYTHON" >> $HERE/env.sh
 fi
-
-# Ensure there is a local pre-commit if there is a git checkout.
-if [ -d .git ]; then
-  if [ ! -f $BIN_DIR/pre-commit ]; then
-    python -m pip install pre-commit
-  fi
-
-  # Ensure the pre-commit hook is installed.
-  if [ ! -f .git/hooks/pre-commit ]; then
-    pre-commit install
-  fi
+echo "Using python $UV_PYTHON"
+uv sync
+uv run --with pip pip install -e .
+echo "Setting up python environment... done."
+
+# Ensure there is a pre-commit hook if there is a git checkout.
+if [ -d .git ] && [ ! -f .git/hooks/pre-commit ]; then
+    uv run pre-commit install
 fi
-
-# Install pymongo and its test deps.
-python -m pip install ".[test]"

.evergreen/teardown-encryption.sh

@@ -7,4 +7,4 @@ if [ -z "${DRIVERS_TOOLS}" ]; then
 fi
 
 bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
-rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz mongocryptd.pid
+rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid

.gitignore

@@ -22,10 +22,9 @@ venv/
 secrets-export.sh
 libmongocrypt.tar.gz
 libmongocrypt/
-libmongocrypt_git/
-hatch_config.toml
 .venv
 expansion.yml
+*expansions.yml
 .evergreen/scripts/env.sh
 
 # Lambda temp files

CONTRIBUTING.md

@@ -16,7 +16,7 @@ be of interest or that has already been addressed.
 
 ## Supported Interpreters
 
-PyMongo supports CPython 3.9+ and PyPy3.9+. Language features not
+PyMongo supports CPython 3.9+ and PyPy3.10+. Language features not
 supported by all interpreters can not be used.
 
 ## Style Guide
@@ -28,7 +28,7 @@ including 4 space indents and 79 character line limits.
 
 -   Avoid backward breaking changes if at all possible.
 -   Write inline documentation for new classes and methods.
--   We use [hatch](https://hatch.pypa.io/dev/) for python environment management and packaging.
+-   We use [uv](https://docs.astral.sh/uv/) for python environment management and packaging.
 -   We use [just](https://just.systems/man/en/) as our task runner.
 -   Write tests and make sure they pass (make sure you have a mongod
     running on the default port, then execute `just test` from the cmd
@@ -194,7 +194,7 @@ the pages will re-render and the browser will automatically refresh.
 ## Running Tests Locally
 
 -   Ensure you have started the appropriate Mongo Server(s).
--   Run `just install` to set up `hatch` in a local virtual environment, or you can manually
+-   Run `just install` to set a local virtual environment, or you can manually
     create a virtual environment and run `pytest` directly.  If you want to use a specific
     version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`.
 -   Run `just test` or `pytest` to run all of the tests.

README.md

@@ -152,11 +152,6 @@ command:
 python -m pip install "pymongo[gssapi,aws,ocsp,snappy,zstd,encryption]"
 ```
 
-Additional dependencies are:
-
--   (to generate documentation or run tests)
-    [hatch](https://hatch.pypa.io/dev/)
-
 ## Examples
 
 Here's a basic example (for more see the *examples* section of the
@@ -201,8 +196,7 @@ ObjectId('4aba160ee23f6b543e000002')
 Documentation is available at
 [pymongo.readthedocs.io](https://pymongo.readthedocs.io/en/stable/).
 
-Documentation can be generated by running **pip install hatch; hatch run doc:build**. Generated
-documentation can be found in the `doc/build/html/` directory.
+See the [contributing guide](./CONTRIBUTING.md#documentation) for how to build the documentation.
 
 ## Learning Resources
 
@@ -213,10 +207,11 @@ Center](https://www.mongodb.com/developer/languages/python/).
 
 ## Testing
 
-The easiest way to run the tests is to run *hatch run test:test** in the root
-of the distribution. For example,
+The easiest way to run the tests is to run the following from the repository root.
 
 ```bash
-pip install hatch
-hatch run test:test
+pip install -e ".[test]"
+pytest
 ```
+
+For more advanced testing scenarios, see the [contributing guide](./CONTRIBUTING.md#running-tests-locally).