From 138573a252620a682a760d5ea4a6e7bfbdf394d3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 08:14:25 -0600 Subject: [PATCH 01/16] PYTHON-4540 Convert libmongocrypt download to python --- .evergreen/run-azurekms-fail-test.sh | 4 +- .evergreen/run-azurekms-test.sh | 3 +- .evergreen/run-gcpkms-test.sh | 3 +- .evergreen/scripts/run-gcpkms-fail-test.sh | 2 - .evergreen/scripts/setup-libmongocrypt.sh | 52 ---------------- .../{setup-tests.py => setup_tests.py} | 61 ++++++++++++++++++- 6 files changed, 63 insertions(+), 62 deletions(-) delete mode 100755 .evergreen/scripts/setup-libmongocrypt.sh rename .evergreen/scripts/{setup-tests.py => setup_tests.py} (83%) diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index 31ca30b3e2..eea84d42a9 100755 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -2,10 +2,8 @@ set -o errexit # Exit the script with error if any of the commands fail HERE=$(dirname ${BASH_SOURCE:-$0}) . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh -PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ - KEY_NAME="${AZUREKMS_KEYNAME}" \ +KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ $HERE/just.sh test-eg bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index 27cb3fb315..8e6b050cb6 100755 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -6,7 +6,6 @@ echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey -LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true @@ -20,7 +19,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \ +AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash .evergreen/just.sh setup-test" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 077ca0cb9f..a430f4e4f9 100755 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -8,7 +8,6 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} -LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true @@ -19,7 +18,7 @@ echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh index 61f5e30ccc..8675c7c242 100755 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -2,7 +2,5 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) . $HERE/env.sh -export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/setup-libmongocrypt.sh b/.evergreen/scripts/setup-libmongocrypt.sh deleted file mode 100755 index 775db07cb0..0000000000 --- a/.evergreen/scripts/setup-libmongocrypt.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -o xtrace - -TARGET="" - -if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin - # PYTHON-2808 Ensure this machine has the CA cert for google KMS. - powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true - TARGET="windows-test" -fi - -if [ "$(uname -s)" = "Darwin" ]; then - TARGET="macos" -fi - -if [ "$(uname -s)" = "Linux" ]; then - rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release) - arch=$(uname -m) - echo "RHEL $rhel_ver $arch" - if [[ $rhel_ver =~ 7 ]]; then - TARGET="rhel-70-64-bit" - elif [[ $rhel_ver =~ 8 ]]; then - if [ "$arch" = "x86_64" ]; then - TARGET="rhel-80-64-bit" - elif [ "$arch" = "arm" ]; then - TARGET="rhel-82-arm64" - fi - fi -fi - -if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then - LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz" -fi - -if [ -z "$LIBMONGOCRYPT_URL" ]; then - echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!" - exit 1 -fi -rm -rf libmongocrypt libmongocrypt.tar.gz -echo "Fetching $LIBMONGOCRYPT_URL..." -curl -O "$LIBMONGOCRYPT_URL" -echo "Fetching $LIBMONGOCRYPT_URL...done" -mkdir libmongocrypt -tar xzf libmongocrypt.tar.gz -C ./libmongocrypt -ls -la libmongocrypt -ls -la libmongocrypt/nocrypto - -if [ "Windows_NT" = "${OS:-''}" ]; then - # libmongocrypt's windows dll is not marked executable. - chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll -fi diff --git a/.evergreen/scripts/setup-tests.py b/.evergreen/scripts/setup_tests.py similarity index 83% rename from .evergreen/scripts/setup-tests.py rename to .evergreen/scripts/setup_tests.py index 4e14ff9830..0713086e53 100644 --- a/.evergreen/scripts/setup-tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -5,11 +5,14 @@ import os import platform import shlex +import shutil import stat import subprocess import sys +import tarfile from pathlib import Path from typing import Any +from urllib import request HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent @@ -92,6 +95,62 @@ def run_command(cmd: str) -> None: LOGGER.info("Running command %s... done.", cmd) +def setup_libmongocrypt(): + target = "" + if os.name == "nt": + # PYTHON-2808 Ensure this machine has the CA cert for google KMS. + if is_set("TEST_FLE_GCP_AUTO"): + run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"') + target = "windows-test" + + elif sys.platform == "darwin": + target = "macos" + + else: + name = "" + version_id = "" + arch = platform.machine() + with open("etc/os-release") as fid: + for line in fid: + if line.startswith("NAME="): + _, _, name = line.strip().split("=") + if line.startswith("VERSION_ID="): + _, _, version = line.strip().split("=") + if name.startswith("Debian"): + target = f"debian{version_id}" + elif name.startswith("Red Hat"): + if version_id.startswith("7"): + target = "rhel-70-64-bit" + elif version_id.startswith("8"): + if arch == "aarch64": + target = "rhel-82-arm64" + else: + target = "rhel-80-64-bit" + + if not is_set("LIBMONGOCRYPT_URL"): + if not target: + raise ValueError("Cannot find libmongocrypt target for current platform!") + url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz" + else: + url = os.environ["LIBMONGOCRYPT_URL"] + + shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True) + + LOGGER.info(f"Fetching {url}...") + with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310 + if response.status == 200: + with tarfile.open("libmongocrypt.tar.gz", fileobj=response) as fid: + fid.extractall(HERE / "libmongocrypt") + LOGGER.info(f"Fetching {url}... done.") + + run_command("ls -la libmongocrypt") + run_command("ls -la libmongocrypt/nocrypto") + + if os.name == "nt": + # libmongocrypt's windows dll is not marked executable. + run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") + + def handle_test_env() -> None: AUTH = os.environ.get("AUTH", "noauth") SSL = os.environ.get("SSL", "nossl") @@ -214,7 +273,7 @@ def handle_test_env() -> None: if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"): # Check for libmongocrypt download. if not (ROOT / "libmongocrypt").exists(): - run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh") + setup_libmongocrypt() # TODO: Test with 'pip install pymongocrypt' UV_ARGS.append("--group pymongocrypt_source") From 2910742d6191230341af217b44f5e8e6526125ea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 08:55:38 -0600 Subject: [PATCH 02/16] fix invocation --- .evergreen/scripts/setup-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 80fc717047..330469711c 100755 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -58,4 +58,4 @@ fi . $ROOT_DIR/.evergreen/utils.sh PYTHON=${PYTHON_BINARY:-$(find_python3)} -$PYTHON $SCRIPT_DIR/setup-tests.py +$PYTHON $SCRIPT_DIR/setup_tests.py From 8f1936092a50c89da01da0882263a646b719594d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 08:59:05 -0600 Subject: [PATCH 03/16] fix invocation --- .evergreen/scripts/setup_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 0713086e53..947f0f8a64 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import io import logging import os import platform @@ -139,8 +140,9 @@ def setup_libmongocrypt(): LOGGER.info(f"Fetching {url}...") with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310 if response.status == 200: - with tarfile.open("libmongocrypt.tar.gz", fileobj=response) as fid: - fid.extractall(HERE / "libmongocrypt") + fileobj = io.BytesIO(response.read()) + with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid: + fid.extractall(Path.cwd() / "libmongocrypt") LOGGER.info(f"Fetching {url}... done.") run_command("ls -la libmongocrypt") From 4f4e10a77cf47d40f7c044d0fb6ebcb3f43b05cc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 09:04:15 -0600 Subject: [PATCH 04/16] fix path --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 947f0f8a64..baa3df9b34 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -111,7 +111,7 @@ def setup_libmongocrypt(): name = "" version_id = "" arch = platform.machine() - with open("etc/os-release") as fid: + with open("/etc/os-release") as fid: for line in fid: if line.startswith("NAME="): _, _, name = line.strip().split("=") From 222bc6672d4e1cbe6af858ebee7212890a412ea7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 09:10:20 -0600 Subject: [PATCH 05/16] fix distro handling --- .evergreen/scripts/setup_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index baa3df9b34..07f33a9d46 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -114,9 +114,9 @@ def setup_libmongocrypt(): with open("/etc/os-release") as fid: for line in fid: if line.startswith("NAME="): - _, _, name = line.strip().split("=") + _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): - _, _, version = line.strip().split("=") + _, _, version = line.strip().partition("=") if name.startswith("Debian"): target = f"debian{version_id}" elif name.startswith("Red Hat"): From 960e4caf2b46cc75cfb31adbe099a647145b232f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 09:15:06 -0600 Subject: [PATCH 06/16] debug --- .evergreen/scripts/setup_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 07f33a9d46..44de6f359d 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -117,6 +117,7 @@ def setup_libmongocrypt(): _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): _, _, version = line.strip().partition("=") + raise ValueError(f"{name=}, {version_id=}, {arch=}") if name.startswith("Debian"): target = f"debian{version_id}" elif name.startswith("Red Hat"): From 8fda5bea354b9f56dd0edcd325c13208b41ebeaa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 09:19:38 -0600 Subject: [PATCH 07/16] debug --- .evergreen/scripts/setup-tests.sh | 1 + .evergreen/scripts/setup_tests.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 330469711c..bdde00632c 100755 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -58,4 +58,5 @@ fi . $ROOT_DIR/.evergreen/utils.sh PYTHON=${PYTHON_BINARY:-$(find_python3)} +cat /etc/os-release $PYTHON $SCRIPT_DIR/setup_tests.py diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 44de6f359d..fcc263a342 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -112,7 +112,7 @@ def setup_libmongocrypt(): version_id = "" arch = platform.machine() with open("/etc/os-release") as fid: - for line in fid: + for line in fid.readlines(): if line.startswith("NAME="): _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): From 3dd1c1e220c556ecb0d0a86b8cb4382a19d18246 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 09:44:21 -0600 Subject: [PATCH 08/16] fix distro handling --- .evergreen/scripts/setup_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index fcc263a342..9682d99338 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -116,8 +116,7 @@ def setup_libmongocrypt(): if line.startswith("NAME="): _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): - _, _, version = line.strip().partition("=") - raise ValueError(f"{name=}, {version_id=}, {arch=}") + _, _, version_id = line.strip().partition("=") if name.startswith("Debian"): target = f"debian{version_id}" elif name.startswith("Red Hat"): From 73fc9ab59b793365134df565a18a41f1aafafdcf Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 10:10:36 -0600 Subject: [PATCH 09/16] debug --- .evergreen/scripts/setup_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 9682d99338..fcf3cf77d2 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -117,6 +117,7 @@ def setup_libmongocrypt(): _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): _, _, version_id = line.strip().partition("=") + raise ValueError(f"{name=} {version_id}=") if name.startswith("Debian"): target = f"debian{version_id}" elif name.startswith("Red Hat"): From ef3e01d7a77741bd4dbd82689bd235e6a8805084 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 10:18:10 -0600 Subject: [PATCH 10/16] fix distro handling --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index fcf3cf77d2..0ed5b9cf14 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -113,11 +113,11 @@ def setup_libmongocrypt(): arch = platform.machine() with open("/etc/os-release") as fid: for line in fid.readlines(): + line = line.replace('"', "") # noqa: PLW2901 if line.startswith("NAME="): _, _, name = line.strip().partition("=") if line.startswith("VERSION_ID="): _, _, version_id = line.strip().partition("=") - raise ValueError(f"{name=} {version_id}=") if name.startswith("Debian"): target = f"debian{version_id}" elif name.startswith("Red Hat"): From 2457b5bf797303b3c9dcced32ef03426dffb4cf7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 10:35:24 -0600 Subject: [PATCH 11/16] debug --- .evergreen/scripts/setup_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 0ed5b9cf14..f0312969b8 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -98,6 +98,7 @@ def run_command(cmd: str) -> None: def setup_libmongocrypt(): target = "" + raise ValueError(f"{sys.platform=}") if os.name == "nt": # PYTHON-2808 Ensure this machine has the CA cert for google KMS. if is_set("TEST_FLE_GCP_AUTO"): From 743fb47a09288ea22b51451d4a4e52367c8070b7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 10:36:19 -0600 Subject: [PATCH 12/16] debug --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index f0312969b8..db18ba3146 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -98,7 +98,7 @@ def run_command(cmd: str) -> None: def setup_libmongocrypt(): target = "" - raise ValueError(f"{sys.platform=}") + raise ValueError(f"{sys.platform=} {os.name=}") if os.name == "nt": # PYTHON-2808 Ensure this machine has the CA cert for google KMS. if is_set("TEST_FLE_GCP_AUTO"): From 16d2c11c1bb76501e72f2e3cb1aef21242221120 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 14 Feb 2025 10:38:40 -0600 Subject: [PATCH 13/16] fix windows and macos handling --- .evergreen/scripts/setup-tests.sh | 1 - .evergreen/scripts/setup_tests.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index bdde00632c..330469711c 100755 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -58,5 +58,4 @@ fi . $ROOT_DIR/.evergreen/utils.sh PYTHON=${PYTHON_BINARY:-$(find_python3)} -cat /etc/os-release $PYTHON $SCRIPT_DIR/setup_tests.py diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index db18ba3146..0ed5b9cf14 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -98,7 +98,6 @@ def run_command(cmd: str) -> None: def setup_libmongocrypt(): target = "" - raise ValueError(f"{sys.platform=} {os.name=}") if os.name == "nt": # PYTHON-2808 Ensure this machine has the CA cert for google KMS. if is_set("TEST_FLE_GCP_AUTO"): From 53542084b3988dc05bfe9402a0db4179e35722cd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Feb 2025 10:17:22 -0600 Subject: [PATCH 14/16] address review --- .evergreen/scripts/setup_tests.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 0ed5b9cf14..bfcb06861e 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -19,6 +19,7 @@ ROOT = HERE.parent.parent ENV_FILE = HERE / "test-env.sh" DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +PLATFORM = "windows" if os.name == "nt" else sys.platform logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) @@ -98,13 +99,13 @@ def run_command(cmd: str) -> None: def setup_libmongocrypt(): target = "" - if os.name == "nt": + if PLATFORM == "windows": # PYTHON-2808 Ensure this machine has the CA cert for google KMS. if is_set("TEST_FLE_GCP_AUTO"): run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"') target = "windows-test" - elif sys.platform == "darwin": + elif PLATFORM == "darwin": target = "macos" else: @@ -149,7 +150,7 @@ def setup_libmongocrypt(): run_command("ls -la libmongocrypt") run_command("ls -la libmongocrypt/nocrypto") - if os.name == "nt": + if PLATFORM == "windows": # libmongocrypt's windows dll is not marked executable. run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") @@ -218,7 +219,7 @@ def handle_test_env() -> None: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if is_set("TEST_ENTERPRISE_AUTH"): - if os.name == "nt": + if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", os.environ["SASL_PASS"]) write_env("GSSAPI_CANONICALIZE", "true") @@ -283,12 +284,12 @@ def handle_test_env() -> None: # Use the nocrypto build to avoid dependency issues with older windows/python versions. BASE = ROOT / "libmongocrypt/nocrypto" - if sys.platform == "linux": + if PLATFORM == "linux": if (BASE / "lib/libmongocrypt.so").exists(): PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so" else: PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so" - elif sys.platform == "darwin": + elif PLATFORM == "darwin": PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib" else: PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll" @@ -306,7 +307,7 @@ def handle_test_env() -> None: if is_set("TEST_CRYPT_SHARED"): CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) - if os.name == "nt": + if PLATFORM == "windows": write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH") else: write_env( From 9918da7897e94174cba4daae86582905f4a7c521 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Feb 2025 10:38:56 -0600 Subject: [PATCH 15/16] add distro handling function --- .evergreen/scripts/setup_tests.py | 45 ++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index bfcb06861e..4698e6ecb5 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import dataclasses import io import logging import os @@ -79,6 +80,13 @@ ) +@dataclasses +class Distro: + name: str + version_id: str + arch: str + + def write_env(name: str, value: Any) -> None: with ENV_FILE.open("a", newline="\n") as fid: # Remove any existing quote chars. @@ -97,6 +105,20 @@ def run_command(cmd: str) -> None: LOGGER.info("Running command %s... done.", cmd) +def get_distro() -> Distro: + name = "" + version_id = "" + arch = platform.machine() + with open("/etc/os-release") as fid: + for line in fid.readlines(): + line = line.replace('"', "") # noqa: PLW2901 + if line.startswith("NAME="): + _, _, name = line.strip().partition("=") + if line.startswith("VERSION_ID="): + _, _, version_id = line.strip().partition("=") + return Distro(name=name, version_id=version_id, arch=arch) + + def setup_libmongocrypt(): target = "" if PLATFORM == "windows": @@ -109,23 +131,14 @@ def setup_libmongocrypt(): target = "macos" else: - name = "" - version_id = "" - arch = platform.machine() - with open("/etc/os-release") as fid: - for line in fid.readlines(): - line = line.replace('"', "") # noqa: PLW2901 - if line.startswith("NAME="): - _, _, name = line.strip().partition("=") - if line.startswith("VERSION_ID="): - _, _, version_id = line.strip().partition("=") - if name.startswith("Debian"): - target = f"debian{version_id}" - elif name.startswith("Red Hat"): - if version_id.startswith("7"): + distro = get_distro() + if distro.name.startswith("Debian"): + target = f"debian{distro.version_id}" + elif distro.name.startswith("Red Hat"): + if distro.version_id.startswith("7"): target = "rhel-70-64-bit" - elif version_id.startswith("8"): - if arch == "aarch64": + elif distro.version_id.startswith("8"): + if distro.arch == "aarch64": target = "rhel-82-arm64" else: target = "rhel-80-64-bit" From 282def934aa8e5b9194e72a67eb04971a8c655e3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Feb 2025 10:55:43 -0600 Subject: [PATCH 16/16] fix handling of dataclass --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 4698e6ecb5..07693d1e99 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -80,7 +80,7 @@ ) -@dataclasses +@dataclasses.dataclass class Distro: name: str version_id: str