diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 8bc758890a..f02b9be878 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -87,32 +87,40 @@ tasks: SUB_TEST_NAME: web-identity PYTHON_VERSION: "3.9" tags: [auth-aws, auth-aws-web-identity] - - name: test-auth-aws-rapid-web-identity-session-name-python3.9 + - name: test-auth-aws-latest-web-identity-session-name-python3.10 commands: - func: run server vars: AUTH_AWS: "1" - VERSION: rapid + VERSION: latest - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity + PYTHON_VERSION: "3.10" AWS_ROLE_SESSION_NAME: test - PYTHON_VERSION: "3.9" tags: [auth-aws, auth-aws-web-identity] - - name: test-auth-aws-latest-ecs-python3.10 + - name: test-auth-aws-latest-eks-python3.11 + commands: + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: auth_aws + SUB_TEST_NAME: eks + PYTHON_VERSION: "3.11" + tags: [auth-aws, auth-aws-eks] + - name: test-auth-aws-latest-ecs-python3.11 commands: + - func: assume ec2 role - func: run server vars: AUTH_AWS: "1" - VERSION: latest - - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws SUB_TEST_NAME: ecs - PYTHON_VERSION: "3.10" + PYTHON_VERSION: "3.11" tags: [auth-aws, auth-aws-ecs] # Backport pr tests diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 673bb111cd..67a84a6cb8 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -98,14 +98,14 @@ buildvariants: tags: [] - name: auth-aws-win64 tasks: - - name: .auth-aws !.auth-aws-ecs + - name: .auth-aws !.auth-aws-ecs !.auth-aws-eks display_name: Auth AWS Win64 run_on: - windows-64-vsMulti-small tags: [] - name: auth-aws-macos tasks: - - name: .auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2 + - name: .auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2 !.auth-aws-eks display_name: Auth AWS macOS run_on: - macos-14 diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 8dc328aab3..82f3a9a4f4 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -76,6 +76,8 @@ EOT rm -rf $DRIVERS_TOOLS BRANCH=master ORG=mongodb-labs +BRANCH=DRIVERS-2945 +ORG=blink1073 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 9f42fb0a4b..f8f6986a52 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -488,10 +488,12 @@ def create_aws_auth_variants(): tasks = [".auth-aws"] tags = [] if host_name == "macos": - tasks = [".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2"] + tasks = [ + ".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2 !.auth-aws-eks" + ] tags = ["pr"] elif host_name == "win64": - tasks = [".auth-aws !.auth-aws-ecs"] + tasks = [".auth-aws !.auth-aws-ecs !.auth-aws-eks"] host = HOSTS[host_name] variant = create_variant( tasks, @@ -741,33 +743,38 @@ def create_aws_tasks(): "env-creds", "session-creds", "web-identity", - "ecs", + "web-identity-session-name", ] + assume_func = FunctionCall(func="assume ec2 role") for version, test_type, python in zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS): base_name = f"test-auth-aws-{version}" base_tags = ["auth-aws"] server_vars = dict(AUTH_AWS="1", VERSION=version) server_func = FunctionCall(func="run server", vars=server_vars) - assume_func = FunctionCall(func="assume ec2 role") - tags = [*base_tags, f"auth-aws-{test_type}"] name = get_task_name(f"{base_name}-{test_type}", python=python) test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, PYTHON_VERSION=python) + if test_type == "web-identity-session-name": + test_type = test_vars["SUB_TEST_NAME"] = "web-identity" # noqa:PLW2901 + test_vars["AWS_ROLE_SESSION_NAME"] = "test" + tags = [*base_tags, f"auth-aws-{test_type}"] test_func = FunctionCall(func="run tests", vars=test_vars) funcs = [server_func, assume_func, test_func] tasks.append(EvgTask(name=name, tags=tags, commands=funcs)) - if test_type == "web-identity": - tags = [*base_tags, "auth-aws-web-identity"] - name = get_task_name(f"{base_name}-web-identity-session-name", python=python) - test_vars = dict( - TEST_NAME="auth_aws", - SUB_TEST_NAME="web-identity", - AWS_ROLE_SESSION_NAME="test", - PYTHON_VERSION=python, - ) - test_func = FunctionCall(func="run tests", vars=test_vars) - funcs = [server_func, assume_func, test_func] - tasks.append(EvgTask(name=name, tags=tags, commands=funcs)) + # These test types use a fixed Python version and the latest server. + for test_type in ["eks", "ecs"]: + tags = ["auth-aws", f"auth-aws-{test_type}"] + base_name = f"test-auth-aws-latest-{test_type}" + python = "3.11" + name = get_task_name(base_name, python=python) + test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, PYTHON_VERSION=python) + test_func = FunctionCall(func="run tests", vars=test_vars) + funcs = [assume_func, test_func] + if test_type == "ecs": + server_vars = dict(AUTH_AWS="1") + server_func = FunctionCall(func="run server", vars=server_vars) + funcs = [assume_func, server_func, test_func] + tasks.append(EvgTask(name=name, tags=tags, commands=funcs)) return tasks diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index ad092983fa..5cc3828ac0 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -197,7 +197,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): name = value - if key.lower() == "python": + if key.lower() == "python" and value is not None: if not value.startswith("pypy"): name = f"Python{value}" else: diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/scripts/run-aws-container-test.sh similarity index 70% rename from .evergreen/run-mongodb-aws-ecs-test.sh rename to .evergreen/scripts/run-aws-container-test.sh index c55c423e49..6d35717f03 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/scripts/run-aws-container-test.sh @@ -13,14 +13,17 @@ fi export MONGODB_URI="$1" if echo "$MONGODB_URI" | grep -q "@"; then - echo "MONGODB_URI unexpectedly contains user credentials in ECS test!"; + echo "MONGODB_URI unexpectedly contains user credentials in container test!"; exit 1 fi # Now we can safely enable xtrace set -o xtrace # Install python with pip. -PYTHON_VER="python3.9" +PYTHON_VER="python3.xx" +apt-get -qq update < /dev/null > /dev/null +apt-get -qq install software-properties-common -y < /dev/null > /dev/null # needed for apt-add-repository +add-apt-repository ppa:deadsnakes/ppa -y || true # this will fail on debian apt-get -qq update < /dev/null > /dev/null apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y < /dev/null > /dev/null diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 5c1ba25a97..736bc0e016 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -11,7 +11,7 @@ from shutil import which import pytest -from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command +from utils import DRIVERS_TOOLS, HERE, LOGGER, ROOT, run_command AUTH = os.environ.get("AUTH", "noauth") SSL = os.environ.get("SSL", "nossl") @@ -159,9 +159,15 @@ def run() -> None: result = main("-E -b doctest doc ./doc/_build/doctest".split()) sys.exit(result) - # Send ecs tests to run remotely. - if TEST_NAME == "auth_aws" and SUB_TEST_NAME == "ecs": - run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh ecs") + # Send ecs and eks tests to run remotely. + if TEST_NAME == "auth_aws" and SUB_TEST_NAME in ["ecs", "eks"]: + target = f"run-mongodb-aws-{SUB_TEST_NAME}-test.sh" + text = (HERE / "run-aws-container-test.sh").read_text() + text = text.replace("python3.xx", os.environ["PYTHON_VERSION"]) + if SUB_TEST_NAME == "eks": + text = text.replace("ecs", "eks") + (HERE.parent / target).write_text(text) + run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh {SUB_TEST_NAME}") return # Send OIDC tests to run remotely. diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 98c382ff60..8b33580592 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -415,11 +415,11 @@ def handle_test_env() -> None: setup_kms(sub_test_name) - if test_name == "auth_aws" and sub_test_name != "ecs-remote": + if test_name == "auth_aws" and sub_test_name not in ["ecs-remote", "eks-remote"]: auth_aws_dir = f"{DRIVERS_TOOLS}/.evergreen/auth_aws" if "AWS_ROLE_SESSION_NAME" in os.environ: write_env("AWS_ROLE_SESSION_NAME") - if sub_test_name != "ecs": + if sub_test_name not in ["ecs", "eks"]: aws_setup = f"{auth_aws_dir}/aws_setup.sh" run_command(f"bash {aws_setup} {sub_test_name}") creds = read_env(f"{auth_aws_dir}/test-env.sh") @@ -427,6 +427,11 @@ def handle_test_env() -> None: write_env(name, value) else: run_command(f"bash {auth_aws_dir}/setup-secrets.sh") + if sub_test_name == "eks": + # Store AWS creds if they were given. + for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]: + if key in os.environ: + write_env(key, os.environ[key]) if test_name == "atlas_connect": get_secrets("drivers/atlas_connect") diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 7a8f9640f8..2560a9224c 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -143,8 +143,8 @@ def get_test_options( raise ValueError(f"Test '{test_name}' requires a sub_test_name") if "auth" in test_name or os.environ.get("AUTH") == "auth": opts.auth = True - # 'auth_aws ecs' shouldn't have extra auth set. - if test_name == "auth_aws" and sub_test_name == "ecs": + # auth_aws ecs or eks shouldn't have extra auth set. + if test_name == "auth_aws" and sub_test_name in ["ecs", "eks"]: opts.auth = False if os.environ.get("SSL") == "ssl": opts.ssl = True diff --git a/.gitignore b/.gitignore index a88a7556e2..fa5be270dd 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ expansion.yml .evergreen/scripts/test-env.sh specifications/ results.json +.evergreen/run-mongodb-aws-eks-test.sh +.evergreen/run-mongodb-aws-ecs-test.sh # Lambda temp files test/lambda/.aws-sam