From 80601f199f202f14cc60f5355a42089e54ce3e9f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 13:30:01 -0600 Subject: [PATCH 01/30] PYTHON-5151 Convert kms tests to use python scripts --- .evergreen/config.yml | 107 +-------------------- .evergreen/run-azurekms-fail-test.sh | 9 -- .evergreen/run-azurekms-test.sh | 27 ------ .evergreen/run-gcpkms-test.sh | 24 ----- .evergreen/scripts/__init__.py | 0 .evergreen/scripts/generate_config.py | 18 ++++ .evergreen/scripts/run-gcpkms-fail-test.sh | 6 -- .evergreen/scripts/run_tests.py | 8 +- .evergreen/scripts/setup_kms.py | 71 ++++++++++++++ .evergreen/scripts/setup_tests.py | 69 +++---------- .evergreen/scripts/teardown-tests.sh | 7 ++ .evergreen/scripts/utils.py | 54 +++++++++++ 12 files changed, 169 insertions(+), 231 deletions(-) delete mode 100755 .evergreen/run-azurekms-fail-test.sh delete mode 100755 .evergreen/run-azurekms-test.sh delete mode 100755 .evergreen/run-gcpkms-test.sh create mode 100644 .evergreen/scripts/__init__.py delete mode 100755 .evergreen/scripts/run-gcpkms-fail-test.sh create mode 100644 .evergreen/scripts/setup_kms.py create mode 100644 .evergreen/scripts/utils.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d00261cc7e..57bca8bfc2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -562,51 +562,6 @@ task_groups: tasks: - ".serverless" - - name: testgcpkms_task_group - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 # 30 minutes - setup_group: - - func: fetch source - - func: setup system - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/create-and-setup-instance.sh - teardown_task: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh - - func: "upload test results" - tasks: - - testgcpkms-task - - - name: testazurekms_task_group - setup_group: - - func: fetch source - - func: setup system - - command: subprocess.exec - params: - binary: bash - env: - AZUREKMS_VMNAME_PREFIX: "PYTHON_DRIVER" - args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh - teardown_group: - - command: subprocess.exec - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/delete-vm.sh - - func: "upload test results" - setup_group_can_fail_task: true - teardown_task_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - testazurekms-task - - name: testazureoidc_task_group setup_group: - func: fetch source @@ -1395,60 +1350,6 @@ tasks: commands: - func: "download and merge coverage" - - name: "testgcpkms-task" - commands: - - command: subprocess.exec - type: test - params: - working_dir: "src" - binary: bash - include_expansions_in_env: ["DRIVERS_TOOLS"] - args: - - .evergreen/run-gcpkms-test.sh - - - name: "testgcpkms-fail-task" - # testgcpkms-fail-task runs in a non-GCE environment. - # It is expected to fail to obtain GCE credentials. - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "latest" - TOPOLOGY: "server" - - command: subprocess.exec - type: test - params: - include_expansions_in_env: ["PYTHON_BINARY"] - working_dir: "src" - binary: "bash" - args: - - .evergreen/scripts/run-gcpkms-fail-test.sh - - - name: testazurekms-task - commands: - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - include_expansions_in_env: ["DRIVERS_TOOLS"] - args: - - .evergreen/run-azurekms-test.sh - - - name: testazurekms-fail-task - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "latest" - TOPOLOGY: "server" - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: src - include_expansions_in_env: ["DRIVERS_TOOLS"] - args: - - .evergreen/run-azurekms-fail-test.sh - - name: "perf-6.0-standalone" tags: ["perf"] commands: @@ -1529,12 +1430,12 @@ buildvariants: run_on: - debian11-small tasks: - - name: testgcpkms_task_group + - name: test-gcpkms batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - - testgcpkms-fail-task - - name: testazurekms_task_group + - name: test-gcpkms-fail + - name: test-azurekms batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - - testazurekms-fail-task + - name: test-azurekms-fail - name: rhel8-test-lambda display_name: FaaS Lambda diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh deleted file mode 100755 index 0804c34d66..0000000000 --- a/.evergreen/run-azurekms-fail-test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -HERE=$(dirname ${BASH_SOURCE:-$0}) -. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh -bash $HERE/just.sh setup-test kms azure-fail -KEY_NAME="${AZUREKMS_KEYNAME}" \ - KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ - $HERE/just.sh test-eg -bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh deleted file mode 100755 index b40b07b019..0000000000 --- a/.evergreen/run-azurekms-test.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -HERE=$(dirname ${BASH_SOURCE:-$0}) -source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh -echo "Copying files ... begin" -export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} -export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} -export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey -# Set up the remote files to test. -git add . -git commit -m "add files" || true -git archive -o /tmp/mongo-python-driver.tgz HEAD -# shellcheck disable=SC2088 -AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh -echo "Copying files ... end" -echo "Untarring file ... begin" -AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh -echo "Untarring file ... end" -echo "Running test ... begin" -AZUREKMS_CMD="bash .evergreen/just.sh setup-test kms azure" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh -AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh -echo "Running test ... end" -bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh deleted file mode 100755 index ee2e8d8a57..0000000000 --- a/.evergreen/run-gcpkms-test.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -HERE=$(dirname ${BASH_SOURCE:-$0}) - -source ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/secrets-export.sh -echo "Copying files ... begin" -export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} -export GCPKMS_PROJECT=${GCPKMS_PROJECT} -export GCPKMS_ZONE=${GCPKMS_ZONE} -export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} -# Set up the remote files to test. -git add . -git commit -m "add files" || true -git archive -o /tmp/mongo-python-driver.tgz HEAD -GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh -echo "Copying files ... end" -echo "Untarring file ... begin" -GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh -echo "Untarring file ... end" -echo "Running test ... begin" -GCPKMS_CMD="bash ./.evergreen/just.sh setup-test kms gcp" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh -GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh -echo "Running test ... end" -bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/scripts/__init__.py b/.evergreen/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index d1c3b92262..73b5709517 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -835,6 +835,24 @@ def create_load_balancer_tasks(): return tasks +def create_kms_tests(): + tasks = [] + for kms_type in ["gcp", "azure"]: + for success in [True, False]: + name = f"test-{kms_type}" + sub_test_name = kms_type + if not success: + name += "-fail" + sub_test_name += " fail" + commands = [] + if not success: + commands.append(FunctionCall(func="bootstrap mongo-orchestration")) + test_vars = dict(TEST_NAME="kms", SUB_TEST_NAME=sub_test_name) + test_func = FunctionCall(func="run tests", vars=test_vars) + commands.append(test_func) + tasks.append(EvgTask(name=name, commands=commands)) + + ################## # Generate Config ################## diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh deleted file mode 100755 index 746ea4103a..0000000000 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -eu -HERE=$(dirname ${BASH_SOURCE:-$0}) -. $HERE/env.sh -./.evergreen/just.sh setup-test kms gcp-fail -bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index e41691ca81..8af26f1cf1 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -1,18 +1,15 @@ from __future__ import annotations import json -import logging import os import platform import shutil import sys from datetime import datetime -from pathlib import Path import pytest +from utils import LOGGER, ROOT -HERE = Path(__file__).absolute().parent -ROOT = HERE.parent.parent AUTH = os.environ.get("AUTH", "noauth") SSL = os.environ.get("SSL", "nossl") UV_ARGS = os.environ.get("UV_ARGS", "") @@ -20,9 +17,6 @@ GREEN_FRAMEWORK = os.environ.get("GREEN_FRAMEWORK") TEST_ARGS = os.environ.get("TEST_ARGS", "").split() -LOGGER = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") - def handle_perf(start_time: datetime): end_time = datetime.now() diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py new file mode 100644 index 0000000000..fdf0af4397 --- /dev/null +++ b/.evergreen/scripts/setup_kms.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import os + +from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env + +TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 + + +def setup_azurekms() -> None: + LOGGER.info("Copying files to Azure VM...") + cmd = f"""AZUREKMS_SRC="{TMP_DRIVER_FILE}" AZUREKMS_DST="~/" \ + {DRIVERS_TOOLS}/.evergreen/csfle/azurekms/copy-file.sh""" + run_command(cmd) + cmd = """AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ + {DRIVERS_TOOLS}/.evergreen/csfle/azurekms/run-command.sh""" + run_command(cmd) + LOGGER.info("Copying files to Azure VM... done.") + + +def setup_gcpkms() -> None: + LOGGER.info("Copying files to GCP VM...") + cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/copy-file.sh" + run_command(cmd) + cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh' + run_command(cmd) + LOGGER.info("Copying files to GCP VM...") + + +def create_archive(): + run_command("git add .", cwd=ROOT) + run_command('git commit -m "add files"', check=False, cwd=ROOT) + run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) + + +def setup_kms(sub_test_name: str, success: bool) -> None: + success = "fail" not in sub_test_name + sub_test_type = sub_test_name.split()[0] + if sub_test_name.startswith("azure"): + write_env("TEST_FLE_AZURE_AUTO") + else: + write_env("TEST_FLE_GCP_AUTO") + + write_env("SUCCESS", success) + + if sub_test_type == "azure": + os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" + + run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup-secrets.sh") + config = read_env(f"{DRIVERS_TOOLS}/csfle/a{sub_test_type}kms/secrets-export.sh") + if success: + run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup.sh") + create_archive() + + if sub_test_name == "azure": + mongodb_uri = setup_azurekms(config) + else: + mongodb_uri = setup_gcpkms(config) + + elif sub_test_type == "azure": + write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) + write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) + + if "@" in mongodb_uri: + raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!") + + write_env("MONGODB_URI", mongodb_uri) + + +if __name__ == "__main__": + setup_kms() diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 78bfad7224..3f511b44da 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -2,29 +2,28 @@ import argparse import base64 -import dataclasses import io import logging import os import platform -import shlex import shutil import stat -import subprocess -import sys import tarfile from pathlib import Path -from typing import Any from urllib import request -HERE = Path(__file__).absolute().parent -ROOT = HERE.parent.parent -ENV_FILE = HERE / "test-env.sh" -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") -PLATFORM = "windows" if os.name == "nt" else sys.platform.lower() - -LOGGER = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") +from utils import ( + DRIVERS_TOOLS, + ENV_FILE, + HERE, + LOGGER, + PLATFORM, + ROOT, + Distro, + read_env, + run_command, + write_env, +) # Passthrough environment variables. PASS_THROUGH_ENV = ["GREEN_FRAMEWORK", "NO_EXT", "MONGODB_API_VERSION"] @@ -69,45 +68,11 @@ GROUP_MAP = dict(mockupdb="mockupdb", perf="perf") -@dataclasses.dataclass -class Distro: - name: str - version_id: str - arch: str - - -def write_env(name: str, value: Any = "1") -> None: - with ENV_FILE.open("a", newline="\n") as fid: - # Remove any existing quote chars. - value = str(value).replace('"', "") - fid.write(f'export {name}="{value}"\n') - - def is_set(var: str) -> bool: value = os.environ.get(var, "") return len(value.strip()) > 0 -def run_command(cmd: str) -> None: - LOGGER.info("Running command %s...", cmd) - subprocess.check_call(shlex.split(cmd)) # noqa: S603 - LOGGER.info("Running command %s... done.", cmd) - - -def read_env(path: Path | str) -> dict[str, Any]: - config = dict() - with Path(path).open() as fid: - for line in fid.readlines(): - if "=" not in line: - continue - name, _, value = line.strip().partition("=") - if value.startswith(('"', "'")): - value = value[1:-1] - name = name.replace("export ", "") - config[name] = value - return config - - def get_options(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -388,15 +353,9 @@ def handle_test_env() -> None: write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}") if test_name == "kms": - if sub_test_name.startswith("azure"): - write_env("TEST_FLE_AZURE_AUTO") - else: - write_env("TEST_FLE_GCP_AUTO") + from .setup_kms import setup_kms - write_env("SUCCESS", "fail" not in sub_test_name) - MONGODB_URI = os.environ.get("MONGODB_URI", "") - if "@" in MONGODB_URI: - raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!") + setup_kms(sub_test_name) if test_name == "ocsp": write_env("CA_FILE", os.environ["CA_FILE"]) diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index be1b88390f..6200042280 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -27,3 +27,10 @@ fi if [ -n "${TEST_LOAD_BALANCER:-}" ]; then bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop fi + +# TODO: move this to a python file. +if [ "TEST_NAME" == "kms" ]; then + if "gcp": + bash ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/teardown.sh + else: + ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/teardown.sh diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py new file mode 100644 index 0000000000..d830275def --- /dev/null +++ b/.evergreen/scripts/utils.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import dataclasses +import logging +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Any + +HERE = Path(__file__).absolute().parent +ROOT = HERE.parent.parent +DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") + +LOGGER = logging.getLogger("test") +logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") +ENV_FILE = HERE / "test-env.sh" +PLATFORM = "windows" if os.name == "nt" else sys.platform.lower() + + +@dataclasses.dataclass +class Distro: + name: str + version_id: str + arch: str + + +def read_env(path: Path | str) -> dict[str, Any]: + config = dict() + with Path(path).open() as fid: + for line in fid.readlines(): + if "=" not in line: + continue + name, _, value = line.strip().partition("=") + if value.startswith(('"', "'")): + value = value[1:-1] + name = name.replace("export ", "") + config[name] = value + return config + + +def write_env(name: str, value: Any = "1") -> None: + with ENV_FILE.open("a", newline="\n") as fid: + # Remove any existing quote chars. + value = str(value).replace('"', "") + fid.write(f'export {name}="{value}"\n') + + +def run_command(cmd: str, **kwargs: Any) -> None: + LOGGER.info("Running command %s...", cmd) + kwargs.setdefault("check", True) + subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 + LOGGER.info("Running command %s... done.", cmd) From ec12424e854d0ab0526df475434425a961fe0c61 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 13:42:22 -0600 Subject: [PATCH 02/30] PYTHON-5151 Convert kms tests to use python scripts --- .evergreen/generated_configs/tasks.yml | 28 ++++++++++++++++++++++++++ .evergreen/scripts/generate_config.py | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 5495ad3470..7eb6556455 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1,4 +1,32 @@ tasks: + # Kms tests + - name: test-gcp + commands: + - func: run tests + vars: + TEST_NAME: kms + SUB_TEST_NAME: gcp + - name: test-gcp-fail + commands: + - func: bootstrap mongo-orchestration + - func: run tests + vars: + TEST_NAME: kms + SUB_TEST_NAME: gcp fail + - name: test-azure + commands: + - func: run tests + vars: + TEST_NAME: kms + SUB_TEST_NAME: azure + - name: test-azure-fail + commands: + - func: bootstrap mongo-orchestration + - func: run tests + vars: + TEST_NAME: kms + SUB_TEST_NAME: azure fail + # Load balancer tests - name: test-load-balancer-auth-ssl commands: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 73b5709517..01f6986b88 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -835,7 +835,7 @@ def create_load_balancer_tasks(): return tasks -def create_kms_tests(): +def create_kms_tasks(): tasks = [] for kms_type in ["gcp", "azure"]: for success in [True, False]: @@ -851,6 +851,7 @@ def create_kms_tests(): test_func = FunctionCall(func="run tests", vars=test_vars) commands.append(test_func) tasks.append(EvgTask(name=name, commands=commands)) + return tasks ################## From e429e09fdf701eeb48029bce48ded2ea2d1bf5ea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 13:43:43 -0600 Subject: [PATCH 03/30] fix task names --- .evergreen/generated_configs/tasks.yml | 8 ++++---- .evergreen/scripts/generate_config.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 7eb6556455..07d3d0c486 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1,25 +1,25 @@ tasks: # Kms tests - - name: test-gcp + - name: test-gcpkms commands: - func: run tests vars: TEST_NAME: kms SUB_TEST_NAME: gcp - - name: test-gcp-fail + - name: test-gcpkms-fail commands: - func: bootstrap mongo-orchestration - func: run tests vars: TEST_NAME: kms SUB_TEST_NAME: gcp fail - - name: test-azure + - name: test-azurekms commands: - func: run tests vars: TEST_NAME: kms SUB_TEST_NAME: azure - - name: test-azure-fail + - name: test-azurekms-fail commands: - func: bootstrap mongo-orchestration - func: run tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 01f6986b88..aee1e35a8d 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -839,7 +839,7 @@ def create_kms_tasks(): tasks = [] for kms_type in ["gcp", "azure"]: for success in [True, False]: - name = f"test-{kms_type}" + name = f"test-{kms_type}kms" sub_test_name = kms_type if not success: name += "-fail" From 984bd87dd8c9ee0b078c04864722e827f538b07b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 14:33:25 -0600 Subject: [PATCH 04/30] fixups --- .evergreen/scripts/bootstrap-mongo-orchestration.sh | 8 ++++---- .evergreen/scripts/setup_tests.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index 5c6387d4b1..8f7d9d0aea 100755 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -34,10 +34,10 @@ if [ -z "${TEST_CRYPT_SHARED:-}" ]; then export SKIP_CRYPT_SHARED=1 fi -MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH:-noauth} \ - SSL=${SSL:-nossl} \ +MONGODB_VERSION=${VERSION:-} \ + TOPOLOGY=${TOPOLOGY:-} \ + AUTH=${AUTH:-} \ + SSL=${SSL:-} \ STORAGE_ENGINE=${STORAGE_ENGINE:-} \ DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \ ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \ diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 3f511b44da..7a1226670c 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -353,7 +353,7 @@ def handle_test_env() -> None: write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}") if test_name == "kms": - from .setup_kms import setup_kms + from setup_kms import setup_kms setup_kms(sub_test_name) From a17f5da3187af26ab1f899acdd714e201106b2d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 14:46:47 -0600 Subject: [PATCH 05/30] refactor --- .evergreen/config.yml | 13 +++--- .evergreen/run-mongodb-aws-ecs-test.sh | 4 +- .evergreen/run-mongodb-oidc-test.sh | 4 +- .evergreen/run-perf-tests.sh | 4 +- .evergreen/run-tests.sh | 2 +- .evergreen/scripts/run-atlas-tests.sh | 4 +- .../scripts/run-enterprise-auth-tests.sh | 4 +- .evergreen/scripts/run-mongodb-aws-test.sh | 4 +- .evergreen/scripts/teardown-tests.sh | 42 ++++++++----------- .evergreen/scripts/teardown_tests.py | 20 +++++++++ CONTRIBUTING.md | 10 ++--- justfile | 6 +-- 12 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 .evergreen/scripts/teardown_tests.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 57bca8bfc2..4c12e8bdb2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -265,13 +265,13 @@ functions: DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS, MONGODB_API_VERSION] binary: bash working_dir: "src" - args: [.evergreen/just.sh, setup-test, "${TEST_NAME}", "${SUB_TEST_NAME}"] + args: [.evergreen/just.sh, setup-tests, "${TEST_NAME}", "${SUB_TEST_NAME}"] - command: subprocess.exec type: test params: working_dir: "src" binary: bash - args: [.evergreen/just.sh, test-eg] + args: [.evergreen/just.sh, run-tests] "run enterprise auth tests": - command: subprocess.exec @@ -443,13 +443,12 @@ functions: binary: bash working_dir: "src" args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh + - ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh - command: subprocess.exec params: - binary: bash - working_dir: "src" - args: - - ${DRIVERS_TOOLS}/.evergreen/ocsp/teardown.sh + binary: bash + working_dir: "src" + args: [.evergreen/just.sh, teardown-tests] - command: subprocess.exec params: binary: bash diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index fc3a092bd4..ef7e0ba333 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -30,5 +30,5 @@ export SET_XTRACE_ON=1 cd src rm -rf .venv rm -f .evergreen/scripts/test-env.sh || true -bash ./.evergreen/just.sh setup-test auth_aws ecs -bash .evergreen/just.sh test-eg +bash ./.evergreen/just.sh setup-tests auth_aws ecs +bash .evergreen/just.sh run-tests diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index c789d6d147..759ac5d2bb 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -29,5 +29,5 @@ else exit 1 fi -COVERAGE=1 bash ./.evergreen/just.sh setup-test auth_oidc -bash ./.evergreen/just.sh test-eg "${@:1}" +COVERAGE=1 bash ./.evergreen/just.sh setup-tests auth_oidc +bash ./.evergreen/just.sh run-tests "${@:1}" diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index 85a82c2a5a..5e423caa23 100755 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -15,5 +15,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 -bash ./.evergreen/just.sh setup-test perf -bash ./.evergreen/just.sh test-eg +bash ./.evergreen/just.sh setup-tests perf +bash ./.evergreen/just.sh run-tests diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index e1b3c779ff..1c453c1d6d 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -20,7 +20,7 @@ if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then echo "Sourcing test inputs" . $SCRIPT_DIR/scripts/test-env.sh else - echo "Missing test inputs, please run 'just setup-test'" + echo "Missing test inputs, please run 'just setup-tests'" exit 1 fi diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 5f9d447c3d..a270d47ab9 100755 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -4,5 +4,5 @@ set +x set -o errexit bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect -bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test atlas -bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-tests atlas +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh reun-tests diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index 21a7fef301..65aafde2df 100755 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -5,5 +5,5 @@ set -eu set +x # Use the default python to bootstrap secrets. bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth -bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-test enterprise_auth -bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-tests enterprise_auth +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh run-tests diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index 917482eaa2..fd38574db8 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -24,5 +24,5 @@ echo "Running MONGODB-AWS authentication tests for $1" # Handle credentials and environment setup. . "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1" -bash ./.evergreen/just.sh setup-test auth_aws $1 -bash ./.evergreen/just.sh test-eg +bash ./.evergreen/just.sh setup-tests auth_aws $1 +bash ./.evergreen/just.sh run-tests diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index 6200042280..ffe88a885b 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -2,35 +2,29 @@ set -eu SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) -ROOT_DIR="$(dirname "$(dirname $SCRIPT_DIR)")" +SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" > /dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname $SCRIPT_DIR)" -# Remove temporary test files. pushd $ROOT_DIR > /dev/null -rm -rf libmongocrypt/ libmongocrypt.tar.gz mongocryptd.pid > /dev/null -popd > /dev/null -if [ ! -f $SCRIPT_DIR/test-env.sh ]; then - exit 0 +# Try to source the env file. +if [ -f $SCRIPT_DIR/scripts/env.sh ]; then + echo "Sourcing env inputs" + . $SCRIPT_DIR/scripts/env.sh +else + echo "Not sourcing env inputs" fi -if [ -f $SCRIPT_DIR/env.sh ]; then - source $SCRIPT_DIR/env.sh -fi - -source $SCRIPT_DIR/test-env.sh -# Shut down csfle servers if applicable -if [ -n "${TEST_ENCRYPTION:-}" ]; then - bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh +# Handle test inputs. +if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then + echo "Sourcing test inputs" + . $SCRIPT_DIR/scripts/test-env.sh +else + echo "Missing test inputs, please run 'just setup-test'" + exit 1 fi -# Shut down load balancer if applicable. -if [ -n "${TEST_LOAD_BALANCER:-}" ]; then - bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop -fi +# Start the test runner. +uv run $SCRIPT_DIR/teardown_tests.py -# TODO: move this to a python file. -if [ "TEST_NAME" == "kms" ]; then - if "gcp": - bash ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/teardown.sh - else: - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/teardown.sh +popd /dev/null diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py new file mode 100644 index 0000000000..93f254cc43 --- /dev/null +++ b/.evergreen/scripts/teardown_tests.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import os + +from utils import DRIVERS_TOOLS, run_command + +TEST_NAME = os.environ["TEST_NAME"] +SUB_TEST_NAME = os.environ["SUB_TEST_NAME"] + +# Shut down csfle servers if applicable +if TEST_NAME == "encryption": + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh") + +# Shut down load balancer if applicable. +elif TEST_NAME == "load-balancer": + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop") + +# Tear down kms VM if applicable. +elif TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/{SUB_TEST_NAME}kms/teardown.sh") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b8b77fc6e..4b1d139684 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -212,17 +212,17 @@ the pages will re-render and the browser will automatically refresh. - Start the servers using `LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh`. - Set up the test using: - `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' just setup-test load-balancer`. + `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' just setup-tests load-balancer`. - Run the tests from the `pymongo` checkout directory using: - `just test-eg`. + `just run-tests`. ## Running Encryption Tests Locally - Clone `drivers-evergreen-tools`: `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. - Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools` -- Run `AWS_PROFILE= just setup-test encryption` after setting up your AWS profile with `aws configure sso`. -- Run the tests with `just test-eg`. -- When done, run `just teardown-test` to clean up. +- Run `AWS_PROFILE= just setup-tests encryption` after setting up your AWS profile with `aws configure sso`. +- Run the tests with `just run-tests`. +- When done, run `just teardown-tests` to clean up. ## Re-sync Spec Tests diff --git a/justfile b/justfile index 5de578ecc7..5a5a05cd5a 100644 --- a/justfile +++ b/justfile @@ -61,13 +61,13 @@ test *args="-v --durations=5 --maxfail=10": {{uv_run}} --extra test pytest {{args}} [group('test')] -test-eg *args: +run-tests *args: bash ./.evergreen/run-tests.sh {{args}} [group('test')] -setup-test *args="": +setup-tests *args="": bash .evergreen/scripts/setup-tests.sh {{args}} [group('test')] -teardown-test: +teardown-tests: bash .evergreen/scripts/teardown-tests.sh From 1a4944a16d9fdf5e7372619c2d21d7fbda83c7dd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:04:13 -0600 Subject: [PATCH 06/30] fix sub_test_name --- .evergreen/generated_configs/tasks.yml | 4 ++-- .evergreen/scripts/generate_config.py | 2 +- .evergreen/scripts/setup_kms.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 07d3d0c486..b7aab80b30 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -12,7 +12,7 @@ tasks: - func: run tests vars: TEST_NAME: kms - SUB_TEST_NAME: gcp fail + SUB_TEST_NAME: gcp-fail - name: test-azurekms commands: - func: run tests @@ -25,7 +25,7 @@ tasks: - func: run tests vars: TEST_NAME: kms - SUB_TEST_NAME: azure fail + SUB_TEST_NAME: azure-fail # Load balancer tests - name: test-load-balancer-auth-ssl diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index aee1e35a8d..4d4d29c6dd 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -843,7 +843,7 @@ def create_kms_tasks(): sub_test_name = kms_type if not success: name += "-fail" - sub_test_name += " fail" + sub_test_name += "-fail" commands = [] if not success: commands.append(FunctionCall(func="bootstrap mongo-orchestration")) diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py index fdf0af4397..26cb0fadfb 100644 --- a/.evergreen/scripts/setup_kms.py +++ b/.evergreen/scripts/setup_kms.py @@ -35,7 +35,7 @@ def create_archive(): def setup_kms(sub_test_name: str, success: bool) -> None: success = "fail" not in sub_test_name - sub_test_type = sub_test_name.split()[0] + sub_test_type = sub_test_name.split("-")[0] if sub_test_name.startswith("azure"): write_env("TEST_FLE_AZURE_AUTO") else: From a5cd2b6301319e90fce950a8d954a382bf62a303 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:12:05 -0600 Subject: [PATCH 07/30] fix kms_setup --- .evergreen/scripts/setup_kms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py index 26cb0fadfb..e445702598 100644 --- a/.evergreen/scripts/setup_kms.py +++ b/.evergreen/scripts/setup_kms.py @@ -33,7 +33,7 @@ def create_archive(): run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) -def setup_kms(sub_test_name: str, success: bool) -> None: +def setup_kms(sub_test_name: str) -> None: success = "fail" not in sub_test_name sub_test_type = sub_test_name.split("-")[0] if sub_test_name.startswith("azure"): From b34f4a8de33b5437f29c62b84998e907d804133e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:14:43 -0600 Subject: [PATCH 08/30] fix kms_setup --- .evergreen/scripts/setup_kms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py index e445702598..df95766b92 100644 --- a/.evergreen/scripts/setup_kms.py +++ b/.evergreen/scripts/setup_kms.py @@ -47,7 +47,7 @@ def setup_kms(sub_test_name: str) -> None: os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup-secrets.sh") - config = read_env(f"{DRIVERS_TOOLS}/csfle/a{sub_test_type}kms/secrets-export.sh") + config = read_env(f"{DRIVERS_TOOLS}/csfle/{sub_test_type}kms/secrets-export.sh") if success: run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup.sh") create_archive() From 5ae7bf8d3d0cc97279224b080f76c47e4e78e128 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:22:19 -0600 Subject: [PATCH 09/30] fix kms_setup --- .evergreen/scripts/setup_kms.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py index df95766b92..a8f0a2fd64 100644 --- a/.evergreen/scripts/setup_kms.py +++ b/.evergreen/scripts/setup_kms.py @@ -5,24 +5,25 @@ from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 +CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" def setup_azurekms() -> None: LOGGER.info("Copying files to Azure VM...") cmd = f"""AZUREKMS_SRC="{TMP_DRIVER_FILE}" AZUREKMS_DST="~/" \ - {DRIVERS_TOOLS}/.evergreen/csfle/azurekms/copy-file.sh""" + {CSFLE_FOLDER}/azurekms/copy-file.sh""" run_command(cmd) cmd = """AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ - {DRIVERS_TOOLS}/.evergreen/csfle/azurekms/run-command.sh""" + {CSFLE_FOLDER}/azurekms/run-command.sh""" run_command(cmd) LOGGER.info("Copying files to Azure VM... done.") def setup_gcpkms() -> None: LOGGER.info("Copying files to GCP VM...") - cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/copy-file.sh" + cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {CSFLE_FOLDER}/gcpkms/copy-file.sh" run_command(cmd) - cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh' + cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {CSFLE_FOLDER}/gcpkms/run-command.sh' run_command(cmd) LOGGER.info("Copying files to GCP VM...") @@ -46,10 +47,10 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_type == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup-secrets.sh") - config = read_env(f"{DRIVERS_TOOLS}/csfle/{sub_test_type}kms/secrets-export.sh") + run_command(f"{CSFLE_FOLDER}/{sub_test_type}kms/setup-secrets.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_type}kms/secrets-export.sh") if success: - run_command(f"{DRIVERS_TOOLS}/.evergreen/csfle/{sub_test_type}kms/setup.sh") + run_command(f"{CSFLE_FOLDER}/{sub_test_type}kms/setup.sh") create_archive() if sub_test_name == "azure": From 206c2a4c99466b8785953f5d0d6cf57ca342d31e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:50:16 -0600 Subject: [PATCH 10/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 97 +++++++++++++++++++++++++++++++ .evergreen/scripts/run_tests.py | 11 +++- .evergreen/scripts/setup_kms.py | 72 ----------------------- .evergreen/scripts/setup_tests.py | 2 +- 4 files changed, 108 insertions(+), 74 deletions(-) create mode 100644 .evergreen/scripts/kms_tester.py delete mode 100644 .evergreen/scripts/setup_kms.py diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py new file mode 100644 index 0000000000..d11c25c878 --- /dev/null +++ b/.evergreen/scripts/kms_tester.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os + +from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env + +TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 +CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" + + +def setup_azure_vm() -> None: + LOGGER.info("Setting up Azure VM...") + cmd = f"""AZUREKMS_SRC="{TMP_DRIVER_FILE}" AZUREKMS_DST="~/" \ + {CSFLE_FOLDER}/azurekms/copy-file.sh""" + run_command(cmd) + cmd = """AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ + {CSFLE_FOLDER}/azurekms/run-command.sh""" + run_command(cmd) + cmd = f"""AZUREKMS_CMD="bash .evergreen/just.sh setup-test kms azure-remote" \ + {CSFLE_FOLDER}/azurekms/run-command.sh""" + run_command(cmd) + LOGGER.info("Setting up Azure VM... done.") + + +def setup_gcp_vm() -> None: + LOGGER.info("Setting up GCP VM...") + cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {CSFLE_FOLDER}/gcpkms/copy-file.sh" + run_command(cmd) + cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {CSFLE_FOLDER}/gcpkms/run-command.sh' + run_command(cmd) + cmd = f'GCPKMS_CMD="bash ./.evergreen/just.sh setup-test kms gcp-remote" {CSFLE_FOLDER}/gcpkms/run-command.sh' + run_command(cmd) + LOGGER.info("Setting up GCP VM...") + + +def create_archive(): + run_command("git add .", cwd=ROOT) + run_command('git commit -m "add files"', check=False, cwd=ROOT) + run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) + + +def setup_kms(sub_test_name: str) -> None: + if "-" in sub_test_name: + sub_test_target, sub_test_type = sub_test_name.split("-")[0] + else: + sub_test_target = sub_test_name + sub_test_type = "" + + assert sub_test_target in ["azure", "kms"] + assert sub_test_type in ["", "remote", "fail"] + success = sub_test_type != "fail" + + if sub_test_target == "azure": + write_env("TEST_FLE_AZURE_AUTO") + else: + write_env("TEST_FLE_GCP_AUTO") + + write_env("SUCCESS", success) + + # For remote tests, there is no further work required. + if sub_test_type == "remote": + return + + if sub_test_target == "azure": + os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" + + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") + if success: + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") + create_archive() + + if sub_test_target == "azure": + setup_azure_vm(config) + else: + setup_gcp_vm(config) + + if sub_test_target == "azure": + write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) + write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) + + +def test_kms_vm(sub_test_name: str) -> None: + if sub_test_name == "azure": + key_name = os.environ["KEY_NAME"] + key_vault_endpoint = os.environ["KEY_VAULT_ENDPOINT"] + cmd = ( + f'AZUREKMS_CMD="KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh test-eg"' + f"{CSFLE_FOLDER}/azurekms/run-command.sh" + ) + else: + cmd = f'GCPKMS_CMD="./.evergreen/just.sh test-eg" {CSFLE_FOLDER}/gcpkms/run-command.sh' + run_command(cmd) + + +if __name__ == "__main__": + setup_kms() diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 8af26f1cf1..1d9ce19775 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -16,6 +16,8 @@ TEST_PERF = os.environ.get("TEST_PERF") GREEN_FRAMEWORK = os.environ.get("GREEN_FRAMEWORK") TEST_ARGS = os.environ.get("TEST_ARGS", "").split() +TEST_NAME = os.environ.get("TEST_NAME") +SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME") def handle_perf(start_time: datetime): @@ -97,7 +99,14 @@ def run() -> None: if TEST_PERF: start_time = datetime.now() - # Run the tests. + # Run remote kms tests. + if TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: + from kms_tester import test_kms_vm + + test_kms_vm(SUB_TEST_NAME) + return + + # Run local tests. pytest.main(TEST_ARGS) # Handle perf test post actions. diff --git a/.evergreen/scripts/setup_kms.py b/.evergreen/scripts/setup_kms.py deleted file mode 100644 index a8f0a2fd64..0000000000 --- a/.evergreen/scripts/setup_kms.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations - -import os - -from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env - -TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 -CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" - - -def setup_azurekms() -> None: - LOGGER.info("Copying files to Azure VM...") - cmd = f"""AZUREKMS_SRC="{TMP_DRIVER_FILE}" AZUREKMS_DST="~/" \ - {CSFLE_FOLDER}/azurekms/copy-file.sh""" - run_command(cmd) - cmd = """AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ - {CSFLE_FOLDER}/azurekms/run-command.sh""" - run_command(cmd) - LOGGER.info("Copying files to Azure VM... done.") - - -def setup_gcpkms() -> None: - LOGGER.info("Copying files to GCP VM...") - cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {CSFLE_FOLDER}/gcpkms/copy-file.sh" - run_command(cmd) - cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {CSFLE_FOLDER}/gcpkms/run-command.sh' - run_command(cmd) - LOGGER.info("Copying files to GCP VM...") - - -def create_archive(): - run_command("git add .", cwd=ROOT) - run_command('git commit -m "add files"', check=False, cwd=ROOT) - run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) - - -def setup_kms(sub_test_name: str) -> None: - success = "fail" not in sub_test_name - sub_test_type = sub_test_name.split("-")[0] - if sub_test_name.startswith("azure"): - write_env("TEST_FLE_AZURE_AUTO") - else: - write_env("TEST_FLE_GCP_AUTO") - - write_env("SUCCESS", success) - - if sub_test_type == "azure": - os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - - run_command(f"{CSFLE_FOLDER}/{sub_test_type}kms/setup-secrets.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_type}kms/secrets-export.sh") - if success: - run_command(f"{CSFLE_FOLDER}/{sub_test_type}kms/setup.sh") - create_archive() - - if sub_test_name == "azure": - mongodb_uri = setup_azurekms(config) - else: - mongodb_uri = setup_gcpkms(config) - - elif sub_test_type == "azure": - write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) - write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) - - if "@" in mongodb_uri: - raise RuntimeError("MONGODB_URI unexpectedly contains user credentials in FLE test!") - - write_env("MONGODB_URI", mongodb_uri) - - -if __name__ == "__main__": - setup_kms() diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 7a1226670c..07be3e833b 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -353,7 +353,7 @@ def handle_test_env() -> None: write_env("LD_LIBRARY_PATH", f"{CRYPT_SHARED_DIR}:${{LD_LIBRARY_PATH:-}}") if test_name == "kms": - from setup_kms import setup_kms + from kms_tester import setup_kms setup_kms(sub_test_name) From 07f6284517afe9653b57c9ba4574175b160e2118 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 15:53:57 -0600 Subject: [PATCH 11/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index d11c25c878..926a702067 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -41,7 +41,7 @@ def create_archive(): def setup_kms(sub_test_name: str) -> None: if "-" in sub_test_name: - sub_test_target, sub_test_type = sub_test_name.split("-")[0] + sub_test_target, sub_test_type = sub_test_name.split("-") else: sub_test_target = sub_test_name sub_test_type = "" From 6a527373d84a3d967bca1608c9cf44cd1fffd08b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 16:17:58 -0600 Subject: [PATCH 12/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 926a702067..3d8c9bca91 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -46,8 +46,8 @@ def setup_kms(sub_test_name: str) -> None: sub_test_target = sub_test_name sub_test_type = "" - assert sub_test_target in ["azure", "kms"] - assert sub_test_type in ["", "remote", "fail"] + assert sub_test_target in ["azure", "gcp"], sub_test_target + assert sub_test_type in ["", "remote", "fail"], sub_test_type success = sub_test_type != "fail" if sub_test_target == "azure": @@ -64,18 +64,18 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") if success: run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") create_archive() if sub_test_target == "azure": - setup_azure_vm(config) + setup_azure_vm() else: - setup_gcp_vm(config) + setup_gcp_vm() if sub_test_target == "azure": + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) From bee26e1505a33197bc1d0d298f3bdb25236f4342 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 16:43:30 -0600 Subject: [PATCH 13/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 60 +++++++++++++++++++------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 3d8c9bca91..56988f2b45 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -10,26 +10,33 @@ def setup_azure_vm() -> None: LOGGER.info("Setting up Azure VM...") - cmd = f"""AZUREKMS_SRC="{TMP_DRIVER_FILE}" AZUREKMS_DST="~/" \ - {CSFLE_FOLDER}/azurekms/copy-file.sh""" - run_command(cmd) - cmd = """AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ - {CSFLE_FOLDER}/azurekms/run-command.sh""" - run_command(cmd) - cmd = f"""AZUREKMS_CMD="bash .evergreen/just.sh setup-test kms azure-remote" \ - {CSFLE_FOLDER}/azurekms/run-command.sh""" - run_command(cmd) + env = os.environ.copy() + env["AZUREKMS_SRC"] = TMP_DRIVER_FILE + env["AZUREKMS_DST"] = "~/" + run_command(f"{CSFLE_FOLDER}/azurekms/copy-file.sh", env=env) + + env = os.environ.copy() + env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz" + run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) + + env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-test kms azure-remote" + run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) LOGGER.info("Setting up Azure VM... done.") -def setup_gcp_vm() -> None: +def setup_gcp_vm(instance_name: str) -> None: LOGGER.info("Setting up GCP VM...") - cmd = f"GCPKMS_SRC={TMP_DRIVER_FILE} GCPKMS_DST=$GCPKMS_INSTANCENAME: {CSFLE_FOLDER}/gcpkms/copy-file.sh" - run_command(cmd) - cmd = f'GCPKMS_CMD="tar xf mongo-python-driver.tgz" {CSFLE_FOLDER}/gcpkms/run-command.sh' - run_command(cmd) - cmd = f'GCPKMS_CMD="bash ./.evergreen/just.sh setup-test kms gcp-remote" {CSFLE_FOLDER}/gcpkms/run-command.sh' - run_command(cmd) + env = os.environ.copy() + env["GCPKMS_SRC"] = TMP_DRIVER_FILE + env["GCPKMS_DST"] = f"{instance_name}:" + run_command(f"{CSFLE_FOLDER}/gcpkms/copy-file.sh", env=env) + + env = os.environ.copy() + env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" + run_command("{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) + + env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-test kms gcp-remote" + run_command("{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) LOGGER.info("Setting up GCP VM...") @@ -64,6 +71,9 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") + if success: run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") create_archive() @@ -71,26 +81,26 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": setup_azure_vm() else: - setup_gcp_vm() + setup_gcp_vm(config["GCPKMS_INSTANCENAME"]) if sub_test_target == "azure": - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) def test_kms_vm(sub_test_name: str) -> None: + env = os.environ.copy() if sub_test_name == "azure": key_name = os.environ["KEY_NAME"] key_vault_endpoint = os.environ["KEY_VAULT_ENDPOINT"] - cmd = ( - f'AZUREKMS_CMD="KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh test-eg"' - f"{CSFLE_FOLDER}/azurekms/run-command.sh" - ) + env[ + "AZUREKMS_CMD" + ] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests' + cmd = f"{CSFLE_FOLDER}/azurekms/run-command.sh" else: - cmd = f'GCPKMS_CMD="./.evergreen/just.sh test-eg" {CSFLE_FOLDER}/gcpkms/run-command.sh' - run_command(cmd) + env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests" + cmd = f"{CSFLE_FOLDER}/gcpkms/run-command.sh" + run_command(cmd, env=env) if __name__ == "__main__": From 489506f1b29920d0280e95a2e837d3f3cd95c453 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 16:50:29 -0600 Subject: [PATCH 14/30] fix teardown --- .evergreen/scripts/teardown-tests.sh | 7 +++---- .evergreen/scripts/teardown_tests.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index ffe88a885b..13b61d4c38 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -10,7 +10,7 @@ pushd $ROOT_DIR > /dev/null # Try to source the env file. if [ -f $SCRIPT_DIR/scripts/env.sh ]; then echo "Sourcing env inputs" - . $SCRIPT_DIR/scripts/env.sh + . $SCRIPT_DIR/env.sh else echo "Not sourcing env inputs" fi @@ -18,13 +18,12 @@ fi # Handle test inputs. if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then echo "Sourcing test inputs" - . $SCRIPT_DIR/scripts/test-env.sh + . $SCRIPT_DIR/test-env.sh else echo "Missing test inputs, please run 'just setup-test'" - exit 1 fi # Start the test runner. uv run $SCRIPT_DIR/teardown_tests.py -popd /dev/null +popd > /dev/null diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index 93f254cc43..4e6e960f99 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -4,8 +4,8 @@ from utils import DRIVERS_TOOLS, run_command -TEST_NAME = os.environ["TEST_NAME"] -SUB_TEST_NAME = os.environ["SUB_TEST_NAME"] +TEST_NAME = os.environ.get("TEST_NAME") +SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME") # Shut down csfle servers if applicable if TEST_NAME == "encryption": From fdc7118e2071732731641d57415af37c06d63613 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 16:55:50 -0600 Subject: [PATCH 15/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 56988f2b45..4d9df5a71f 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -8,14 +8,14 @@ CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" -def setup_azure_vm() -> None: +def setup_azure_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up Azure VM...") - env = os.environ.copy() + env = base_env.copy() env["AZUREKMS_SRC"] = TMP_DRIVER_FILE env["AZUREKMS_DST"] = "~/" run_command(f"{CSFLE_FOLDER}/azurekms/copy-file.sh", env=env) - env = os.environ.copy() + env = base_env.copy() env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz" run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) @@ -24,14 +24,14 @@ def setup_azure_vm() -> None: LOGGER.info("Setting up Azure VM... done.") -def setup_gcp_vm(instance_name: str) -> None: +def setup_gcp_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up GCP VM...") - env = os.environ.copy() + env = base_env.copy() env["GCPKMS_SRC"] = TMP_DRIVER_FILE - env["GCPKMS_DST"] = f"{instance_name}:" + env["GCPKMS_DST"] = f"{env['GCPKMS_INSTANCENAME']}:" run_command(f"{CSFLE_FOLDER}/gcpkms/copy-file.sh", env=env) - env = os.environ.copy() + env = base_env.copy() env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" run_command("{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) @@ -72,18 +72,23 @@ def setup_kms(sub_test_name: str) -> None: os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") if success: - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") create_archive() + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") + config = read_env(f"{CSFLE_FOLDER}/gcpkms/secrets-export.sh") + base_env = os.environ.copy() + for key, value in config.items(): + base_env[key] = str(value) + if sub_test_target == "azure": - setup_azure_vm() + setup_azure_vm(base_env) else: - setup_gcp_vm(config["GCPKMS_INSTANCENAME"]) + setup_gcp_vm(base_env) if sub_test_target == "azure": + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) From 9df90f07a922b1172ff3510f7479b62d7d8a9cbe Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 17:07:09 -0600 Subject: [PATCH 16/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 31 ++++++++++++++-------------- .evergreen/scripts/teardown-tests.sh | 4 ++-- .evergreen/scripts/teardown_tests.py | 12 ++++++++--- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 4d9df5a71f..8129ef1669 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -8,7 +8,7 @@ CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" -def setup_azure_vm(base_env: dict[str, str]) -> None: +def _setup_azure_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up Azure VM...") env = base_env.copy() env["AZUREKMS_SRC"] = TMP_DRIVER_FILE @@ -24,7 +24,7 @@ def setup_azure_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up Azure VM... done.") -def setup_gcp_vm(base_env: dict[str, str]) -> None: +def _setup_gcp_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up GCP VM...") env = base_env.copy() env["GCPKMS_SRC"] = TMP_DRIVER_FILE @@ -33,14 +33,14 @@ def setup_gcp_vm(base_env: dict[str, str]) -> None: env = base_env.copy() env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" - run_command("{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) + run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-test kms gcp-remote" - run_command("{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) + run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) LOGGER.info("Setting up GCP VM...") -def create_archive(): +def _create_archive(): run_command("git add .", cwd=ROOT) run_command('git commit -m "add files"', check=False, cwd=ROOT) run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) @@ -68,26 +68,23 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_type == "remote": return - if sub_test_target == "azure": - os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") - if success: - create_archive() - + _create_archive() + if sub_test_target == "azure": + os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") - config = read_env(f"{CSFLE_FOLDER}/gcpkms/secrets-export.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}/secrets-export.sh") base_env = os.environ.copy() for key, value in config.items(): base_env[key] = str(value) if sub_test_target == "azure": - setup_azure_vm(base_env) + _setup_azure_vm(base_env) else: - setup_gcp_vm(base_env) + _setup_gcp_vm(base_env) if sub_test_target == "azure": + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) @@ -108,5 +105,9 @@ def test_kms_vm(sub_test_name: str) -> None: run_command(cmd, env=env) +def teardown_kms(sub_test_name: str) -> None: + run_command(f"bash {CSFLE_FOLDER}/{sub_test_name}kms/teardown.sh") + + if __name__ == "__main__": setup_kms() diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index 13b61d4c38..653c015d07 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -8,7 +8,7 @@ ROOT_DIR="$(dirname $SCRIPT_DIR)" pushd $ROOT_DIR > /dev/null # Try to source the env file. -if [ -f $SCRIPT_DIR/scripts/env.sh ]; then +if [ -f $SCRIPT_DIR/env.sh ]; then echo "Sourcing env inputs" . $SCRIPT_DIR/env.sh else @@ -16,7 +16,7 @@ else fi # Handle test inputs. -if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then +if [ -f $SCRIPT_DIR/test-env.sh ]; then echo "Sourcing test inputs" . $SCRIPT_DIR/test-env.sh else diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index 4e6e960f99..fc1a937de0 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -2,11 +2,13 @@ import os -from utils import DRIVERS_TOOLS, run_command +from utils import DRIVERS_TOOLS, LOGGER, run_command -TEST_NAME = os.environ.get("TEST_NAME") +TEST_NAME = os.environ.get("TEST_NAME", "unconfigured") SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME") +LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'...") + # Shut down csfle servers if applicable if TEST_NAME == "encryption": run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh") @@ -17,4 +19,8 @@ # Tear down kms VM if applicable. elif TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/{SUB_TEST_NAME}kms/teardown.sh") + from kms_tester import teardown_kms + + teardown_kms(SUB_TEST_NAME) + +LOGGER.info(f"Tearing down tests of type '{TEST_NAME}'... done.") From 969d203e75fbd02449cf5f26a8a9c942521d6678 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 18:53:57 -0600 Subject: [PATCH 17/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 2 +- .evergreen/scripts/setup_tests.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 8129ef1669..55c5120605 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -73,7 +73,7 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}/secrets-export.sh") + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") base_env = os.environ.copy() for key, value in config.items(): base_env[key] = str(value) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 07be3e833b..580de8cc53 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -203,6 +203,7 @@ def handle_test_env() -> None: # Set an environment variable for the test name and sub test name. write_env(f"TEST_{test_name.upper()}") + write_env("TEST_NAME", test_name) write_env("SUB_TEST_NAME", sub_test_name) # Handle pass through env vars. From 264c7a66fd98cc0d46357673300bd66830cd8951 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 19:04:45 -0600 Subject: [PATCH 18/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 4 ++-- .evergreen/scripts/setup_tests.py | 12 +++++++++--- .evergreen/scripts/teardown-tests.sh | 2 +- test/asynchronous/test_on_demand_csfle.py | 2 +- test/test_on_demand_csfle.py | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 55c5120605..c2f3f1f8b7 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -19,7 +19,7 @@ def _setup_azure_vm(base_env: dict[str, str]) -> None: env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz" run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) - env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-test kms azure-remote" + env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote" run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) LOGGER.info("Setting up Azure VM... done.") @@ -35,7 +35,7 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None: env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) - env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-test kms gcp-remote" + env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote" run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) LOGGER.info("Setting up GCP VM...") diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 580de8cc53..615f07320a 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -40,7 +40,7 @@ "encryption": "encryption", "enterprise_auth": "auth", "index_management": "index_management", - "kms": "csfle", + "kms": "kms", "load_balancer": "load_balancer", "mockupdb": "mockupdb", "pyopenssl": "", @@ -77,8 +77,14 @@ def get_options(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument("test_name", choices=sorted(TEST_SUITE_MAP), nargs="?", default="default") - parser.add_argument("sub_test_name", nargs="?") + parser.add_argument( + "test_name", + choices=sorted(TEST_SUITE_MAP), + nargs="?", + default="default", + help="The name of the test suite to set up, typically the same name as a pytest marker.", + ) + parser.add_argument("sub_test_name", nargs="?", help="The sub test name, for example 'azure'") parser.add_argument( "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level" ) diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index 653c015d07..9ee948c0f7 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -20,7 +20,7 @@ if [ -f $SCRIPT_DIR/test-env.sh ]; then echo "Sourcing test inputs" . $SCRIPT_DIR/test-env.sh else - echo "Missing test inputs, please run 'just setup-test'" + echo "Missing test inputs, please run 'just setup-tests'" fi # Start the test runner. diff --git a/test/asynchronous/test_on_demand_csfle.py b/test/asynchronous/test_on_demand_csfle.py index 617e2ed8d6..55394ddeb8 100644 --- a/test/asynchronous/test_on_demand_csfle.py +++ b/test/asynchronous/test_on_demand_csfle.py @@ -34,7 +34,7 @@ _IS_SYNC = False -pytestmark = pytest.mark.csfle +pytestmark = pytest.mark.kms class TestonDemandGCPCredentials(AsyncIntegrationTest): diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index 023d44f641..648e46815a 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -34,7 +34,7 @@ _IS_SYNC = True -pytestmark = pytest.mark.csfle +pytestmark = pytest.mark.kms class TestonDemandGCPCredentials(IntegrationTest): From 9f7ec22fddbf47a849f503db30503671d2a56d2b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 19:07:23 -0600 Subject: [PATCH 19/30] fix kms_setup --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b86e9df6ad..c898169895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ markers = [ "data_lake: tests that rely on atlas data lake", "perf: benchmark tests", "index_management: index management tests", - "csfle: client-side field-level encryption tests", + "kms: client-side field-level encryption tests using kms", "encryption: encryption tests", "load_balancer: load balancer tests", "mockupdb: tests that rely on mockupdb", From 097779f520683f2bb64af96c9d30ee6464aebb7b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 19:19:18 -0600 Subject: [PATCH 20/30] fix kms_setup --- .evergreen/scripts/cleanup.sh | 3 --- .evergreen/scripts/kms_tester.py | 19 +++++++++++-------- .evergreen/scripts/setup-dev-env.sh | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index a1fd92f04d..c58d2163dd 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -1,7 +1,4 @@ #!/bin/bash -if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then - bash .evergreen/teardown-encryption.sh -fi rm -rf "${DRIVERS_TOOLS}" || true rm -f ./secrets-export.sh || true diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index c2f3f1f8b7..3f71be650f 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -40,12 +40,19 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up GCP VM...") -def _create_archive(): +def _create_archive() -> None: run_command("git add .", cwd=ROOT) run_command('git commit -m "add files"', check=False, cwd=ROOT) run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) +def _load_kms_config(sub_test_target: str) -> None: + config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") + base_env = os.environ.copy() + for key, value in config.items(): + base_env[key] = str(value) + + def setup_kms(sub_test_name: str) -> None: if "-" in sub_test_name: sub_test_target, sub_test_type = sub_test_name.split("-") @@ -73,10 +80,7 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") - base_env = os.environ.copy() - for key, value in config.items(): - base_env[key] = str(value) + base_env = _load_kms_config(sub_test_target) if sub_test_target == "azure": _setup_azure_vm(base_env) @@ -91,17 +95,16 @@ def setup_kms(sub_test_name: str) -> None: def test_kms_vm(sub_test_name: str) -> None: - env = os.environ.copy() + env = _load_kms_config(sub_test_name) if sub_test_name == "azure": key_name = os.environ["KEY_NAME"] key_vault_endpoint = os.environ["KEY_VAULT_ENDPOINT"] env[ "AZUREKMS_CMD" ] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests' - cmd = f"{CSFLE_FOLDER}/azurekms/run-command.sh" else: env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests" - cmd = f"{CSFLE_FOLDER}/gcpkms/run-command.sh" + cmd = f"{CSFLE_FOLDER}/{sub_test_name}kms/run-command.sh" run_command(cmd, env=env) diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 04a377a2d2..f158c71320 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eux +set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" From 2e8cebd817d7aa5e71bbf1a91e84a06cf7c99594 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 19:29:50 -0600 Subject: [PATCH 21/30] fix kms_setup --- .evergreen/scripts/kms_tester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 3f71be650f..af85f1e9f8 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -46,11 +46,12 @@ def _create_archive() -> None: run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT) -def _load_kms_config(sub_test_target: str) -> None: +def _load_kms_config(sub_test_target: str) -> dict[str, str]: config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") base_env = os.environ.copy() for key, value in config.items(): base_env[key] = str(value) + return base_env def setup_kms(sub_test_name: str) -> None: From 7b09bfc37ded1d7719dd12c02d51c88a48345e42 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 19:38:45 -0600 Subject: [PATCH 22/30] fix azure setup --- .evergreen/scripts/kms_tester.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index af85f1e9f8..2f9ca597db 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -80,6 +80,7 @@ def setup_kms(sub_test_name: str) -> None: _create_archive() if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" + run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") base_env = _load_kms_config(sub_test_target) @@ -93,6 +94,7 @@ def setup_kms(sub_test_name: str) -> None: config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) + write_env("AZUREKMS_VMNAME", config["AZUREKMS_KEYNAME"]) def test_kms_vm(sub_test_name: str) -> None: From cd7f043c6d941c798a549591db1e184bf09429aa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 20:51:59 -0600 Subject: [PATCH 23/30] fix azure setup --- .evergreen/scripts/kms_tester.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 2f9ca597db..6a5f5079d3 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -5,7 +5,7 @@ from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 -CSFLE_FOLDER = f"{DRIVERS_TOOLS}/.evergreen/csfle" +CSFLE_DIR = f"{DRIVERS_TOOLS}/.evergreen/csfle" def _setup_azure_vm(base_env: dict[str, str]) -> None: @@ -13,14 +13,14 @@ def _setup_azure_vm(base_env: dict[str, str]) -> None: env = base_env.copy() env["AZUREKMS_SRC"] = TMP_DRIVER_FILE env["AZUREKMS_DST"] = "~/" - run_command(f"{CSFLE_FOLDER}/azurekms/copy-file.sh", env=env) + run_command(f"{CSFLE_DIR}/azurekms/copy-file.sh", env=env) env = base_env.copy() env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz" - run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) + run_command(f"{CSFLE_DIR}/azurekms/run-command.sh", env=env) env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote" - run_command(f"{CSFLE_FOLDER}/azurekms/run-command.sh", env=env) + run_command(f"{CSFLE_DIR}/azurekms/run-command.sh", env=env) LOGGER.info("Setting up Azure VM... done.") @@ -29,14 +29,14 @@ def _setup_gcp_vm(base_env: dict[str, str]) -> None: env = base_env.copy() env["GCPKMS_SRC"] = TMP_DRIVER_FILE env["GCPKMS_DST"] = f"{env['GCPKMS_INSTANCENAME']}:" - run_command(f"{CSFLE_FOLDER}/gcpkms/copy-file.sh", env=env) + run_command(f"{CSFLE_DIR}/gcpkms/copy-file.sh", env=env) env = base_env.copy() env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" - run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) + run_command(f"{CSFLE_DIR}/gcpkms/run-command.sh", env=env) env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote" - run_command(f"{CSFLE_FOLDER}/gcpkms/run-command.sh", env=env) + run_command(f"{CSFLE_DIR}/gcpkms/run-command.sh", env=env) LOGGER.info("Setting up GCP VM...") @@ -47,7 +47,7 @@ def _create_archive() -> None: def _load_kms_config(sub_test_target: str) -> dict[str, str]: - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") + config = read_env(f"{CSFLE_DIR}/{sub_test_target}kms/secrets-export.sh") base_env = os.environ.copy() for key, value in config.items(): base_env[key] = str(value) @@ -64,6 +64,7 @@ def setup_kms(sub_test_name: str) -> None: assert sub_test_target in ["azure", "gcp"], sub_test_target assert sub_test_type in ["", "remote", "fail"], sub_test_type success = sub_test_type != "fail" + kms_dir = f"{CSFLE_DIR}/{sub_test_target}kms" if sub_test_target == "azure": write_env("TEST_FLE_AZURE_AUTO") @@ -81,7 +82,7 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup.sh") + run_command("setup.sh", cwd=kms_dir) base_env = _load_kms_config(sub_test_target) if sub_test_target == "azure": @@ -90,8 +91,8 @@ def setup_kms(sub_test_name: str) -> None: _setup_gcp_vm(base_env) if sub_test_target == "azure": - run_command(f"{CSFLE_FOLDER}/{sub_test_target}kms/setup-secrets.sh") - config = read_env(f"{CSFLE_FOLDER}/{sub_test_target}kms/secrets-export.sh") + run_command("setup-secrets.sh", cwd=kms_dir) + config = read_env(f"{kms_dir}/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) write_env("AZUREKMS_VMNAME", config["AZUREKMS_KEYNAME"]) @@ -107,12 +108,12 @@ def test_kms_vm(sub_test_name: str) -> None: ] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests' else: env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests" - cmd = f"{CSFLE_FOLDER}/{sub_test_name}kms/run-command.sh" + cmd = f"{CSFLE_DIR}/{sub_test_name}kms/run-command.sh" run_command(cmd, env=env) def teardown_kms(sub_test_name: str) -> None: - run_command(f"bash {CSFLE_FOLDER}/{sub_test_name}kms/teardown.sh") + run_command(f"{CSFLE_DIR}/{sub_test_name}kms/teardown.sh") if __name__ == "__main__": From 50c9ee0f4681226b2e6de453287fe793ac130efa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 06:17:44 -0600 Subject: [PATCH 24/30] cleanup --- .evergreen/scripts/kms_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 6a5f5079d3..08c043336f 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -82,7 +82,7 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": os.environ["AZUREKMS_VMNAME_PREFIX"] = "PYTHON_DRIVER" - run_command("setup.sh", cwd=kms_dir) + run_command("./setup.sh", cwd=kms_dir) base_env = _load_kms_config(sub_test_target) if sub_test_target == "azure": @@ -91,7 +91,7 @@ def setup_kms(sub_test_name: str) -> None: _setup_gcp_vm(base_env) if sub_test_target == "azure": - run_command("setup-secrets.sh", cwd=kms_dir) + run_command("./setup-secrets.sh", cwd=kms_dir) config = read_env(f"{kms_dir}/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) From 76cc63f6428eb3ac99046523bee8ee91e5b9a210 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 06:29:02 -0600 Subject: [PATCH 25/30] add files --- .evergreen/scripts/kms_tester.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 08c043336f..aea8ed5db2 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -5,38 +5,43 @@ from utils import DRIVERS_TOOLS, LOGGER, ROOT, read_env, run_command, write_env TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 -CSFLE_DIR = f"{DRIVERS_TOOLS}/.evergreen/csfle" +DIRS = dict( + gcp=f"{DRIVERS_TOOLS}/.evergreen/csfle/gcpkms", + azure=f"{DRIVERS_TOOLS}/.evergreen/csfle/azurekms", +) def _setup_azure_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up Azure VM...") + azure_dir = DIRS["azure"] env = base_env.copy() env["AZUREKMS_SRC"] = TMP_DRIVER_FILE env["AZUREKMS_DST"] = "~/" - run_command(f"{CSFLE_DIR}/azurekms/copy-file.sh", env=env) + run_command(f"{azure_dir}/copy-file.sh", env=env) env = base_env.copy() env["AZUREKMS_CMD"] = "tar xf mongo-python-driver.tgz" - run_command(f"{CSFLE_DIR}/azurekms/run-command.sh", env=env) + run_command(f"{azure_dir}/run-command.sh", env=env) env["AZUREKMS_CMD"] = "bash .evergreen/just.sh setup-tests kms azure-remote" - run_command(f"{CSFLE_DIR}/azurekms/run-command.sh", env=env) + run_command(f"{azure_dir}/run-command.sh", env=env) LOGGER.info("Setting up Azure VM... done.") def _setup_gcp_vm(base_env: dict[str, str]) -> None: LOGGER.info("Setting up GCP VM...") + gcp_dir = DIRS["gcp"] env = base_env.copy() env["GCPKMS_SRC"] = TMP_DRIVER_FILE env["GCPKMS_DST"] = f"{env['GCPKMS_INSTANCENAME']}:" - run_command(f"{CSFLE_DIR}/gcpkms/copy-file.sh", env=env) + run_command(f"{gcp_dir}/copy-file.sh", env=env) env = base_env.copy() env["GCPKMS_CMD"] = "tar xf mongo-python-driver.tgz" - run_command(f"{CSFLE_DIR}/gcpkms/run-command.sh", env=env) + run_command(f"{gcp_dir}/run-command.sh", env=env) env["GCPKMS_CMD"] = "bash ./.evergreen/just.sh setup-tests kms gcp-remote" - run_command(f"{CSFLE_DIR}/gcpkms/run-command.sh", env=env) + run_command(f"{gcp_dir}/run-command.sh", env=env) LOGGER.info("Setting up GCP VM...") @@ -47,7 +52,8 @@ def _create_archive() -> None: def _load_kms_config(sub_test_target: str) -> dict[str, str]: - config = read_env(f"{CSFLE_DIR}/{sub_test_target}kms/secrets-export.sh") + target_dir = DIRS[sub_test_target] + config = read_env(f"{target_dir}/secrets-export.sh") base_env = os.environ.copy() for key, value in config.items(): base_env[key] = str(value) @@ -64,7 +70,7 @@ def setup_kms(sub_test_name: str) -> None: assert sub_test_target in ["azure", "gcp"], sub_test_target assert sub_test_type in ["", "remote", "fail"], sub_test_type success = sub_test_type != "fail" - kms_dir = f"{CSFLE_DIR}/{sub_test_target}kms" + kms_dir = DIRS[sub_test_target] if sub_test_target == "azure": write_env("TEST_FLE_AZURE_AUTO") @@ -108,12 +114,12 @@ def test_kms_vm(sub_test_name: str) -> None: ] = f'KEY_NAME="{key_name}" KEY_VAULT_ENDPOINT="{key_vault_endpoint}" bash ./.evergreen/just.sh run-tests' else: env["GCPKMS_CMD"] = "./.evergreen/just.sh run-tests" - cmd = f"{CSFLE_DIR}/{sub_test_name}kms/run-command.sh" + cmd = f"{DIRS[sub_test_name]}/run-command.sh" run_command(cmd, env=env) def teardown_kms(sub_test_name: str) -> None: - run_command(f"{CSFLE_DIR}/{sub_test_name}kms/teardown.sh") + run_command(f"{DIRS[sub_test_name]}/teardown.sh") if __name__ == "__main__": From 6168d3ab00ce5e1300de14696391b51b50c86b8e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 06:40:48 -0600 Subject: [PATCH 26/30] fix azure setup --- .evergreen/scripts/kms_tester.py | 8 +++++--- .evergreen/scripts/run_tests.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index aea8ed5db2..25aaf4919d 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -83,6 +83,9 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_type == "remote": return + if sub_test_target == "azure": + run_command("./setup-secrets.sh", cwd=kms_dir) + if success: _create_archive() if sub_test_target == "azure": @@ -97,14 +100,13 @@ def setup_kms(sub_test_name: str) -> None: _setup_gcp_vm(base_env) if sub_test_target == "azure": - run_command("./setup-secrets.sh", cwd=kms_dir) config = read_env(f"{kms_dir}/secrets-export.sh") write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) - write_env("AZUREKMS_VMNAME", config["AZUREKMS_KEYNAME"]) + write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"]) -def test_kms_vm(sub_test_name: str) -> None: +def test_kms_remote(sub_test_name: str) -> None: env = _load_kms_config(sub_test_name) if sub_test_name == "azure": key_name = os.environ["KEY_NAME"] diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 1d9ce19775..ceae46d343 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -101,9 +101,9 @@ def run() -> None: # Run remote kms tests. if TEST_NAME == "kms" and SUB_TEST_NAME in ["azure", "gcp"]: - from kms_tester import test_kms_vm + from kms_tester import test_kms_remote - test_kms_vm(SUB_TEST_NAME) + test_kms_remote(SUB_TEST_NAME) return # Run local tests. From 31d84a21559313d842a6b34c57bea038c574f37c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 06:58:25 -0600 Subject: [PATCH 27/30] fix azure setup --- .evergreen/scripts/kms_tester.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index 25aaf4919d..f9ed967447 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -101,9 +101,11 @@ def setup_kms(sub_test_name: str) -> None: if sub_test_target == "azure": config = read_env(f"{kms_dir}/secrets-export.sh") - write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) - write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) - write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"]) + if success: + write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"]) + else: + write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) + write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) def test_kms_remote(sub_test_name: str) -> None: From f51901b861e80b363d030b2a5d20cf4f825ba4d4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 07:19:17 -0600 Subject: [PATCH 28/30] fix azure setup --- .evergreen/scripts/kms_tester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/kms_tester.py b/.evergreen/scripts/kms_tester.py index f9ed967447..d38ec3a69e 100644 --- a/.evergreen/scripts/kms_tester.py +++ b/.evergreen/scripts/kms_tester.py @@ -103,9 +103,9 @@ def setup_kms(sub_test_name: str) -> None: config = read_env(f"{kms_dir}/secrets-export.sh") if success: write_env("AZUREKMS_VMNAME", config["AZUREKMS_VMNAME"]) - else: - write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) - write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) + + write_env("KEY_NAME", config["AZUREKMS_KEYNAME"]) + write_env("KEY_VAULT_ENDPOINT", config["AZUREKMS_KEYVAULTENDPOINT"]) def test_kms_remote(sub_test_name: str) -> None: From 60c6745b57946e84b9294229e703f0964c3aa667 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 07:47:07 -0600 Subject: [PATCH 29/30] fix typo --- .evergreen/scripts/run-atlas-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index a270d47ab9..99968063bd 100755 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -5,4 +5,4 @@ set +x set -o errexit bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh setup-tests atlas -bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh reun-tests +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh run-tests From 5be69fbad88e025e6e7d18622054d659fa5a4b79 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 25 Feb 2025 14:38:04 -0600 Subject: [PATCH 30/30] Update .evergreen/scripts/teardown-tests.sh Co-authored-by: Noah Stapp --- .evergreen/scripts/teardown-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index 9ee948c0f7..cd705c6b35 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -23,7 +23,7 @@ else echo "Missing test inputs, please run 'just setup-tests'" fi -# Start the test runner. +# Teardown the test runner. uv run $SCRIPT_DIR/teardown_tests.py popd > /dev/null