diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ac89270d84..5c0e2983ea 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -42,7 +42,7 @@ functions: # Make an evergreen expansion file with dynamic values - command: subprocess.exec params: - include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "SKIP_HATCH", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"] + include_expansions_in_env: ["is_patch", "project", "version_id", "AUTH", "SSL", "test_encryption", "test_encryption_pyopenssl", "test_crypt_shared", "test_pyopenssl", "SETDEFAULTENCODING", "test_loadbalancer", "test_serverless", "SKIP_CSOT_TESTS", "MONGODB_STARTED", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", "COVERAGE", "COMPRESSORS", "TEST_SUITES", "MONGODB_API_VERSION", "skip_crypt_shared", "VERSION", "TOPOLOGY", "STORAGE_ENGINE", "ORCHESTRATION_FILE", "REQUIRE_API_VERSION", "LOAD_BALANCER", "skip_web_identity_auth_test", "skip_ECS_auth_test"] binary: bash working_dir: "src" args: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 928347f567..226f4238f2 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -8,7 +8,6 @@ buildvariants: - rhel79-small batchtime: 10080 expansions: - SKIP_HATCH: "true" PYTHON_BINARY: /opt/python/3.9/bin/python3 - name: other-hosts-rhel9-fips tasks: @@ -17,8 +16,6 @@ buildvariants: run_on: - rhel92-fips batchtime: 10080 - expansions: - SKIP_HATCH: "true" - name: other-hosts-rhel8-zseries tasks: - name: .6.0 .standalone !.sync_async @@ -26,8 +23,6 @@ buildvariants: run_on: - rhel8-zseries-small batchtime: 10080 - expansions: - SKIP_HATCH: "true" - name: other-hosts-rhel8-power8 tasks: - name: .6.0 .standalone !.sync_async @@ -35,8 +30,6 @@ buildvariants: run_on: - rhel8-power-small batchtime: 10080 - expansions: - SKIP_HATCH: "true" - name: other-hosts-rhel8-arm64 tasks: - name: .6.0 .standalone !.sync_async @@ -44,8 +37,6 @@ buildvariants: run_on: - rhel82-arm64-small batchtime: 10080 - expansions: - SKIP_HATCH: "true" # Atlas connect tests - name: atlas-connect-rhel8-python3.9 diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh index 98cd9ed734..c01dfcd19e 100755 --- a/.evergreen/hatch.sh +++ b/.evergreen/hatch.sh @@ -1,45 +1,5 @@ #!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -x +set -eu -. .evergreen/utils.sh - -if [ -z "$PYTHON_BINARY" ]; then - PYTHON_BINARY=$(find_python3) -fi - -# Check if we should skip hatch and run the tests directly. -if [ -n "$SKIP_HATCH" ]; then - ENV_NAME=testenv-$RANDOM - createvirtualenv "$PYTHON_BINARY" $ENV_NAME - # shellcheck disable=SC2064 - trap "deactivate; rm -rf $ENV_NAME" EXIT HUP - python -m pip install -e ".[test]" - run_hatch() { - bash ./.evergreen/run-tests.sh - } -else # Set up virtualenv before installing hatch - # Use a random venv name because the encryption tasks run this script multiple times in the same run. - ENV_NAME=hatchenv-$RANDOM - createvirtualenv "$PYTHON_BINARY" $ENV_NAME - # shellcheck disable=SC2064 - trap "deactivate; rm -rf $ENV_NAME" EXIT HUP - python -m pip install -q hatch - - # Ensure hatch does not write to user or global locations. - touch hatch_config.toml - HATCH_CONFIG=$(pwd)/hatch_config.toml - if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") - fi - export HATCH_CONFIG - hatch config restore - hatch config set dirs.data "$(pwd)/.hatch/data" - hatch config set dirs.cache "$(pwd)/.hatch/cache" - - run_hatch() { - python -m hatch run "$@" - } -fi - -run_hatch "${@:1}" +. .evergreen/scripts/ensure-hatch.sh +hatch run "$@" diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 9f4bcdbb59..4c0541a4e2 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -8,6 +8,9 @@ cp ${PROJECT_DIRECTORY}/test/certificates/* ${DRIVERS_TOOLS}/.evergreen/x509gen/ # Replace MongoOrchestration's client certificate. cp ${PROJECT_DIRECTORY}/test/certificates/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem +# Ensure hatch is installed. +bash ${PROJECT_DIRECTORY}/scripts/ensure-hatch.sh + if [ -w /etc/hosts ]; then SUDO="" else diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 3905a08764..3189a6cc6c 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -22,13 +22,13 @@ set -o xtrace # Install python with pip. PYTHON_VER="python3.9" -apt-get update -apt-get install $PYTHON_VER python3-pip build-essential $PYTHON_VER-dev -y +apt-get -qq update < /dev/null > /dev/null +apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y < /dev/null > /dev/null export PYTHON_BINARY=$PYTHON_VER export TEST_AUTH_AWS=1 export AUTH="auth" export SET_XTRACE_ON=1 cd src -$PYTHON_BINARY -m pip install -q --user hatch +rm -rf .venv bash .evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 3c0a0436de..313f4c3c92 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -11,11 +11,13 @@ fi PROJECT_DIRECTORY="$(pwd)" DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) + CARGO_HOME=$(cygpath -m $CARGO_HOME) fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -47,13 +49,13 @@ export NO_EXT="${NO_EXT:-}" export COVERAGE="${COVERAGE:-}" export COMPRESSORS="${COMPRESSORS:-}" export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}" -export SKIP_HATCH="${SKIP_HATCH:-}" export skip_crypt_shared="${skip_crypt_shared:-}" export STORAGE_ENGINE="${STORAGE_ENGINE:-}" export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}" export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}" export skip_ECS_auth_test="${skip_ECS_auth_test:-}" +export CARGO_HOME="$CARGO_HOME" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" # shellcheck disable=SC2154 diff --git a/.evergreen/scripts/ensure-hatch.sh b/.evergreen/scripts/ensure-hatch.sh new file mode 100755 index 0000000000..a57b705127 --- /dev/null +++ b/.evergreen/scripts/ensure-hatch.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -eu + +# Ensure hatch is available. +if [ ! -x "$(command -v hatch)" ]; then + # Install a virtual env with "hatch" + # Ensure there is a python venv. + . .evergreen/utils.sh + + if [ -z "${PYTHON_BINARY:-}" ]; then + PYTHON_BINARY=$(find_python3) + fi + VENV_DIR=.venv + if [ ! -d $VENV_DIR ]; then + echo "Creating virtual environment..." + createvirtualenv "$PYTHON_BINARY" .venv + echo "Creating virtual environment... done." + fi + if [ -f $VENV_DIR/Scripts/activate ]; then + . $VENV_DIR/Scripts/activate + else + . $VENV_DIR/bin/activate + fi + + python --version + + echo "Installing hatch..." + python -m pip install -U pip + python -m pip install hatch || { + # Install rust and try again. + CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} + # Handle paths on Windows. + if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin + CARGO_HOME=$(cygpath -m $CARGO_HOME) + fi + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + source "${CARGO_HOME}/env" + python -m pip install hatch + } + # Ensure hatch does not write to user or global locations. + touch hatch_config.toml + HATCH_CONFIG=$(pwd)/hatch_config.toml + if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin + HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") + fi + export HATCH_CONFIG + hatch config restore + hatch config set dirs.data "$(pwd)/.hatch/data" + hatch config set dirs.cache "$(pwd)/.hatch/cache" + + echo "Installing hatch... done." +fi +hatch --version diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index b7187b50db..c7f55fa946 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -744,7 +744,6 @@ def create_aws_auth_variants(): def create_alternative_hosts_variants(): - expansions = dict(SKIP_HATCH="true") batchtime = BATCHTIME_WEEK variants = [] @@ -752,11 +751,10 @@ def create_alternative_hosts_variants(): variants.append( create_variant( [".5.0 .standalone !.sync_async"], - get_display_name("OpenSSL 1.0.2", host, python=CPYTHONS[0], **expansions), + get_display_name("OpenSSL 1.0.2", host, python=CPYTHONS[0]), host=host, python=CPYTHONS[0], batchtime=batchtime, - expansions=expansions, ) ) @@ -765,8 +763,7 @@ def create_alternative_hosts_variants(): variants.append( create_variant( [".6.0 .standalone !.sync_async"], - display_name=get_display_name("Other hosts", host, **expansions), - expansions=expansions, + display_name=get_display_name("Other hosts", host), batchtime=batchtime, host=host, ) diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index 31371ead45..11f8db22e1 100755 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -2,5 +2,6 @@ # Disable xtrace for security reasons (just in case it was accidentally set). set +x -bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth +# Use the default python to bootstrap secrets. +PYTHON_BINARY="" bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 495db83e70..6986a0bbee 100755 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -51,5 +51,4 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ TEST_SUITES=${TEST_SUITES:-} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - SKIP_HATCH=${SKIP_HATCH} \ bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg diff --git a/.gitignore b/.gitignore index 69dd20efa3..e4587125e8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ secrets-export.sh libmongocrypt.tar.gz libmongocrypt/ libmongocrypt_git/ +hatch_config.toml +.venv # Lambda temp files test/lambda/.aws-sam