From fb17cf42c52b562cd6996e9414223585f793efc4 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 09:34:31 -0700 Subject: [PATCH 01/22] PYTHON-3912 Migrate Enterprise Auth Tests to AWS Secrets Vault --- .evergreen/config.yml | 33 +++++++++------------------------ .evergreen/run-tests.sh | 2 +- .gitignore | 1 - test/test_client.py | 3 +++ 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a1ea000098..1de253d645 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -477,6 +477,7 @@ functions: fi if [ -n "${test_loadbalancer}" ]; then export TEST_LOADBALANCER=1 + export LOAD_BALANCER=1 export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" fi @@ -488,12 +489,6 @@ functions: export SINGLE_MONGOS_LB_URI="${MONGODB_URI}" export MULTI_MONGOS_LB_URI="${MONGODB_URI}" fi - if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then - export TEST_INDEX_MANAGEMENT=1 - export MONGODB_URI="${TEST_INDEX_URI}" - export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}" - export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" - fi export PYTHON_BINARY=${PYTHON_BINARY} if [ -z "$PYTHON_BINARY" ]; then @@ -511,31 +506,21 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run enterprise auth tests": - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: "src" - script: | - cat < prepare_enterprise_auth.sh - export SASL_HOST='${sasl_host}' - export SASL_PORT='${sasl_port}' - export SASL_USER='${sasl_user}' - export SASL_PASS='${sasl_pass}' - export SASL_DB='${sasl_db}' - export PRINCIPAL='${principal}' - export GSSAPI_DB='${gssapi_db}' - export KEYTAB_BASE64='${keytab_base64}' - EOT + role_arn: ${aws_test_secrets_role} - command: shell.exec type: test params: + add_expansions_to_env: true working_dir: "src" script: | - # Disable xtrace (just in case it was accidentally set). + # Disable xtrace for security reasons (just in case it was accidentally set). set +x - . ./prepare_enterprise_auth.sh - rm -f ./prepare_enterprise_auth.sh + set -o errexit + set -o xtrace + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth + PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ TEST_ENTERPRISE_AUTH=1 \ diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index daeb9b1dcd..c15a2be985 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -234,7 +234,7 @@ fi echo "Running $AUTH tests over $SSL with python $PYTHON" python -c 'import sys; print(sys.version)' -# Try to source exported AWS Secrets +# Try to source exported AWS Secrets, fixing Windows line endings if [ -f ./secrets-export.sh ]; then source ./secrets-export.sh fi diff --git a/.gitignore b/.gitignore index 2663e31757..77483d26b2 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ mongocryptd.pid .idea/ .nova/ venv/ -secrets-expansion.yml secrets-export.sh # Lambda temp files diff --git a/test/test_client.py b/test/test_client.py index 24f4603b27..5d5208043d 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1758,6 +1758,9 @@ def _test_handshake(self, env_vars, expected_env): metadata = copy.deepcopy(_METADATA) if expected_env is not None: metadata["env"] = expected_env + + if "AWS_REGION" not in env_vars: + os.environ["AWS_REGION"] = "" with rs_or_single_client(serverSelectionTimeoutMS=10000) as client: client.admin.command("ping") options = client._MongoClient__options From 6072dc5c808c1954b55915bce73ffedb2598f6d2 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 10:12:54 -0700 Subject: [PATCH 02/22] Remove expansions from env --- .evergreen/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1de253d645..fc51065d1e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,7 +512,6 @@ functions: - command: shell.exec type: test params: - add_expansions_to_env: true working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). From 0d4b027702475d665d5a383a83151d152c3f7988 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 10:25:24 -0700 Subject: [PATCH 03/22] Explicitly set DRIVERS_TOOLS --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fc51065d1e..d994263209 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -522,6 +522,7 @@ functions: PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ + DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ TEST_ENTERPRISE_AUTH=1 \ AUTH=auth \ bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg From 32b80b1c2fd38d97a42bb78fb6621be3ce478acd Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 10:37:36 -0700 Subject: [PATCH 04/22] Use bash -c --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d994263209..20b77f9956 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -525,7 +525,7 @@ functions: DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ TEST_ENTERPRISE_AUTH=1 \ AUTH=auth \ - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg + bash -c ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run atlas tests": - command: ec2.assume_role From 5078f8ce4a26e3c5842a0e383406380de92f2c78 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 10:45:38 -0700 Subject: [PATCH 05/22] Debug --- .evergreen/config.yml | 2 +- tox.ini | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 20b77f9956..d994263209 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -525,7 +525,7 @@ functions: DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ TEST_ENTERPRISE_AUTH=1 \ AUTH=auth \ - bash -c ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run atlas tests": - command: ec2.assume_role diff --git a/tox.ini b/tox.ini index 55545c54f0..93c2bb7895 100644 --- a/tox.ini +++ b/tox.ini @@ -157,5 +157,7 @@ deps = PyYAML boto3 passenv = * +allowlist_externals = printenv commands = + printenv python {env:DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.py {posargs} From 0f5c67c48bd8a35b0e6f0e142000497d0f6a22d2 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 10:53:29 -0700 Subject: [PATCH 06/22] Debug --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d994263209..c38a5db978 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,6 +512,7 @@ functions: - command: shell.exec type: test params: + add_expansions_to_env: true working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). From 54b40895c7c5e3502166f4ba80b7caef23cfe165 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 11:01:53 -0700 Subject: [PATCH 07/22] more testing --- tox.ini | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 93c2bb7895..8b7ba95ddc 100644 --- a/tox.ini +++ b/tox.ini @@ -156,7 +156,13 @@ commands = deps = PyYAML boto3 -passenv = * +passenv = + DRIVERS_TOOLS + AWS_SESSION_TOKEN + AWS_ROLE_ARN + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN allowlist_externals = printenv commands = printenv From d7d3470c2cecee84a737ddc0390d858a463e32f5 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 11:11:12 -0700 Subject: [PATCH 08/22] Explicitly allow test-eg env vars --- tox.ini | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8b7ba95ddc..7fe8f88fc7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,30 @@ commands = [testenv:test-eg] description = run tests using run-tests.sh Evergreen script -passenv = * +passenv = + SET_XTRACE_ON + AUTH + SSL + GREEN_FRAMEWORK + C_EXTENSIONS + COVERAGE + COMPRESSORS + LIBMONGOCRYPT_URL + TEST_DATA_LAKE + TEST_ENCRYPTION + TEST_CRYPT_SHARED + TEST_SERVERLESS + TEST_LOADBALANCER + TEST_FLE_AZURE_AUTO + TEST_FLE_GCP_AUTO + TEST_PYOPENSSL + TEST_ENTERPRISE_AUTH + TEST_AUTH_AWS + TEST_AUTH_OIDC + TEST_PERF + TEST_OCSP + TEST_ENCRYPTION_PYOPENSSL + TEST_ATLAS deps = pytest>=7 setuptools From 4e6a6630ba6005a479b1a86a9428a1914820dd9f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 11:23:50 -0700 Subject: [PATCH 09/22] more pass testing --- tox.ini | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 7fe8f88fc7..1a0c3a3c76 100644 --- a/tox.ini +++ b/tox.ini @@ -75,6 +75,14 @@ passenv = TEST_OCSP TEST_ENCRYPTION_PYOPENSSL TEST_ATLAS + SASL_HOST=ldaptest.10gen.cc + SASL_PORT=27017 + SASL_USER=drivers + SASL_PASS=powerbook17 + SASL_DB=\$external + PRINCIPAL=drivers@LDAPTEST.10GEN.CC + GSSAPI_DB=kerberos + KEYTAB_BASE64=BQIAAAA7AAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABcAELMC+Gi0iyFaNJuDmUtwFTwAAAA7AAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABcAELMC+Gi0iyFaNJuDmUtwFTwAAABLAAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABIAIDxSS2skk4KWoKncYEIkJXhDJV8aLm0gYkc8ufWTPRAhAAAAUAABABFMREFQVEVTVC4xMEdFTi5DQwAMc2NocsO2ZGluZ2VyAAAAAViHlNUBABIAIOn/Sbke7jmhTBzduQXoqN0nWBgpIZKbAEVNlPifg6c9 deps = pytest>=7 setuptools @@ -186,7 +194,5 @@ passenv = AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN -allowlist_externals = printenv commands = - printenv python {env:DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.py {posargs} From ac604123beae02aafbb93d692e3b54b2ec21ad64 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 11:49:59 -0700 Subject: [PATCH 10/22] more pass testing --- tox.ini | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 1a0c3a3c76..72ba7b2d47 100644 --- a/tox.ini +++ b/tox.ini @@ -75,14 +75,15 @@ passenv = TEST_OCSP TEST_ENCRYPTION_PYOPENSSL TEST_ATLAS - SASL_HOST=ldaptest.10gen.cc - SASL_PORT=27017 - SASL_USER=drivers - SASL_PASS=powerbook17 - SASL_DB=\$external - PRINCIPAL=drivers@LDAPTEST.10GEN.CC - GSSAPI_DB=kerberos - KEYTAB_BASE64=BQIAAAA7AAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABcAELMC+Gi0iyFaNJuDmUtwFTwAAAA7AAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABcAELMC+Gi0iyFaNJuDmUtwFTwAAABLAAEAEUxEQVBURVNULjEwR0VOLkNDAAdkcml2ZXJzAAAAAViHlNUBABIAIDxSS2skk4KWoKncYEIkJXhDJV8aLm0gYkc8ufWTPRAhAAAAUAABABFMREFQVEVTVC4xMEdFTi5DQwAMc2NocsO2ZGluZ2VyAAAAAViHlNUBABIAIOn/Sbke7jmhTBzduQXoqN0nWBgpIZKbAEVNlPifg6c9 + SASL_HOST + SASL_PORT + SASL_USER + SASL_PASS + SASL_DB + PRINCIPAL + GSSAPI_DB + KEYTAB_BASE64 + PROJECT_DIRECTORY deps = pytest>=7 setuptools From ffb39bc5751cc741ba3c455e1cf909a3cb98d96a Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 12:13:06 -0700 Subject: [PATCH 11/22] more cebugging --- tox.ini | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/tox.ini b/tox.ini index 72ba7b2d47..0c59318b25 100644 --- a/tox.ini +++ b/tox.ini @@ -51,45 +51,15 @@ commands = [testenv:test-eg] description = run tests using run-tests.sh Evergreen script -passenv = - SET_XTRACE_ON - AUTH - SSL - GREEN_FRAMEWORK - C_EXTENSIONS - COVERAGE - COMPRESSORS - LIBMONGOCRYPT_URL - TEST_DATA_LAKE - TEST_ENCRYPTION - TEST_CRYPT_SHARED - TEST_SERVERLESS - TEST_LOADBALANCER - TEST_FLE_AZURE_AUTO - TEST_FLE_GCP_AUTO - TEST_PYOPENSSL - TEST_ENTERPRISE_AUTH - TEST_AUTH_AWS - TEST_AUTH_OIDC - TEST_PERF - TEST_OCSP - TEST_ENCRYPTION_PYOPENSSL - TEST_ATLAS - SASL_HOST - SASL_PORT - SASL_USER - SASL_PASS - SASL_DB - PRINCIPAL - GSSAPI_DB - KEYTAB_BASE64 - PROJECT_DIRECTORY +passenv = * deps = pytest>=7 setuptools allowlist_externals = bash + printenv commands = + printenv bash ./.evergreen/run-tests.sh {posargs} [testenv:lint] From b59b44f975e5549b3ca3b359de2b6a549dce75f1 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 12:45:30 -0700 Subject: [PATCH 12/22] more cebugging --- .evergreen/run-tests.sh | 1 + tox.ini | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index c15a2be985..cf79fb480a 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -67,6 +67,7 @@ if [ -n "$TEST_ENTERPRISE_AUTH" ]; then echo "Writing keytab" echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab echo "Running kinit" + python -c "import os;print(sorted([(k, v[:2]) for k, v in os.environ.items()]))" kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} fi echo "Setting GSSAPI variables" diff --git a/tox.ini b/tox.ini index 0c59318b25..87741d08c0 100644 --- a/tox.ini +++ b/tox.ini @@ -57,9 +57,7 @@ deps = setuptools allowlist_externals = bash - printenv commands = - printenv bash ./.evergreen/run-tests.sh {posargs} [testenv:lint] From a63f0f45dc506444de2f793b5d00b15d8dbf7ab3 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 13:42:38 -0700 Subject: [PATCH 13/22] =?UTF-8?q?explicit=20passing=20of=20env=20vars?= =?UTF-8?q?=EF=9D=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .evergreen/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c38a5db978..c4a092141c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,14 +512,13 @@ functions: - command: shell.exec type: test params: - add_expansions_to_env: true working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). set +x set -o errexit set -o xtrace - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth + DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ From cf34cce0e5f0f6ddf5984562f21908528dee159a Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 14:03:21 -0700 Subject: [PATCH 14/22] test --- tox.ini | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 87741d08c0..55545c54f0 100644 --- a/tox.ini +++ b/tox.ini @@ -156,12 +156,6 @@ commands = deps = PyYAML boto3 -passenv = - DRIVERS_TOOLS - AWS_SESSION_TOKEN - AWS_ROLE_ARN - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN +passenv = * commands = python {env:DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.py {posargs} From fcd8fd432a46d9ee85b91efd61dbebcdadc33d8c Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 14:26:57 -0700 Subject: [PATCH 15/22] test --- .evergreen/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c4a092141c..1e09389794 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -518,7 +518,15 @@ functions: set +x set -o errexit set -o xtrace - DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth + export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + export AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + + DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ From a7d2d51dd8dd91ab61874a2a7f0ed75b7de9b59f Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 14:44:07 -0700 Subject: [PATCH 16/22] Source secrets first --- .evergreen/config.yml | 4 ---- .evergreen/run-tests.sh | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1e09389794..6fe2e07d3a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -518,9 +518,6 @@ functions: set +x set -o errexit set -o xtrace - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - export AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ @@ -530,7 +527,6 @@ functions: PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ - DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ TEST_ENTERPRISE_AUTH=1 \ AUTH=auth \ bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index cf79fb480a..ddb000335c 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -41,6 +41,11 @@ PYTHON=$(which python) python -c "import sys; sys.exit(sys.prefix == sys.base_prefix)" || (echo "Not inside a virtual env!"; exit 1) +# Try to source exported AWS Secrets +if [ -f ./secrets-export.sh ]; then + source ./secrets-export.sh +fi + if [ "$AUTH" != "noauth" ]; then if [ ! -z "$TEST_DATA_LAKE" ]; then export DB_USER="mhuser" @@ -235,10 +240,6 @@ fi echo "Running $AUTH tests over $SSL with python $PYTHON" python -c 'import sys; print(sys.version)' -# Try to source exported AWS Secrets, fixing Windows line endings -if [ -f ./secrets-export.sh ]; then - source ./secrets-export.sh -fi # Run the tests, and store the results in Evergreen compatible XUnit XML # files in the xunit-results/ directory. From 2a2cfe9d22c21dc30491c88418c9bfc6ff5872a6 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 15:15:34 -0700 Subject: [PATCH 17/22] try again --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6fe2e07d3a..266d249f7d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -512,6 +512,7 @@ functions: - command: shell.exec type: test params: + add_expansions_to_env: true working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). From 87421834aa414db6125d0a82936724be4e44fb9e Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 16 Aug 2023 15:48:42 -0700 Subject: [PATCH 18/22] Cleanup --- .evergreen/config.yml | 6 +----- .evergreen/run-tests.sh | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 266d249f7d..88bfb5f9fb 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -520,11 +520,7 @@ functions: set -o errexit set -o xtrace - DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ - AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ - AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ - AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index ddb000335c..5db4640401 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -72,7 +72,6 @@ if [ -n "$TEST_ENTERPRISE_AUTH" ]; then echo "Writing keytab" echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab echo "Running kinit" - python -c "import os;print(sorted([(k, v[:2]) for k, v in os.environ.items()]))" kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} fi echo "Setting GSSAPI variables" From 5d8c1c502ad7372bc6ada7d2cccbf4cdd76a33ae Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 17 Aug 2023 09:03:37 -0700 Subject: [PATCH 19/22] Fixes --- .evergreen/config.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 88bfb5f9fb..fc165a9e65 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -477,7 +477,6 @@ functions: fi if [ -n "${test_loadbalancer}" ]; then export TEST_LOADBALANCER=1 - export LOAD_BALANCER=1 export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" fi @@ -489,6 +488,12 @@ functions: export SINGLE_MONGOS_LB_URI="${MONGODB_URI}" export MULTI_MONGOS_LB_URI="${MONGODB_URI}" fi + if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then + export TEST_INDEX_MANAGEMENT=1 + export MONGODB_URI="${TEST_INDEX_URI}" + export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}" + export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" + fi export PYTHON_BINARY=${PYTHON_BINARY} if [ -z "$PYTHON_BINARY" ]; then @@ -512,15 +517,24 @@ functions: - command: shell.exec type: test params: - add_expansions_to_env: true + include_expansions_in_env: [ + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_ROLE_ARN + AWS_SESSION_TOKEN + DRIVERS_TOOLS + ] working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). set +x - set -o errexit - set -o xtrace - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth + DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + AWS_ROLE_ARN="${AWS_ROLE_ARN}" \ + AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ PYTHON_BINARY="${PYTHON_BINARY}" \ From 259af9c0ee7f2cecab640542837a8b26d43d47ba Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 17 Aug 2023 14:31:14 -0700 Subject: [PATCH 20/22] Use include_expansions_in_env for atlas tests --- .evergreen/config.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fc165a9e65..10082c4c7c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -549,16 +549,29 @@ functions: - command: shell.exec type: test params: - add_expansions_to_env: true + include_expansions_in_env: [ + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_ROLE_ARN + AWS_SESSION_TOKEN + DRIVERS_TOOLS + ] working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). set +x - set -o errexit - set -o xtrace - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/atlas_connect - TEST_ATLAS=1 bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg + DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + AWS_ROLE_ARN="${AWS_ROLE_ARN}" \ + AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/atlas_connect + + PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \ + PYTHON_BINARY="${PYTHON_BINARY}" \ + TEST_ATLAS=1 \ + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "add aws auth variables to file": - command: shell.exec From 14986e7585c22b4db3730371a51933b46c8ea778 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 17 Aug 2023 14:42:15 -0700 Subject: [PATCH 21/22] Remove include_expansions_in_env since it doesn't work --- .evergreen/config.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 10082c4c7c..4567fa8ad3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -517,13 +517,6 @@ functions: - command: shell.exec type: test params: - include_expansions_in_env: [ - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_ROLE_ARN - AWS_SESSION_TOKEN - DRIVERS_TOOLS - ] working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). @@ -549,13 +542,6 @@ functions: - command: shell.exec type: test params: - include_expansions_in_env: [ - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_ROLE_ARN - AWS_SESSION_TOKEN - DRIVERS_TOOLS - ] working_dir: "src" script: | # Disable xtrace for security reasons (just in case it was accidentally set). From d3abbf72527a151a952aae9778cb5d0e614bf4a5 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Fri, 18 Aug 2023 09:12:11 -0700 Subject: [PATCH 22/22] Remove AWS_ROLE_ARN --- .evergreen/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4567fa8ad3..a8b87b7853 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -525,7 +525,6 @@ functions: DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ - AWS_ROLE_ARN="${AWS_ROLE_ARN}" \ AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/enterprise_auth @@ -550,7 +549,6 @@ functions: DRIVERS_TOOLS="${DRIVERS_TOOLS}" \ AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ - AWS_ROLE_ARN="${AWS_ROLE_ARN}" \ AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \ bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m aws-secrets -- drivers/atlas_connect