From 9149b842eae7215b58ea8cdb364e84d8edce7986 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 31 Oct 2023 11:26:01 +0100 Subject: [PATCH 01/14] Switch to ubuntu 2004 --- .evergreen/config.yml | 7 +++++-- .evergreen/config/axes.yml.erb | 5 ++++- .evergreen/config/standard.yml.erb | 2 +- spec/shared | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e41d305254..b9aab5ba4b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -972,7 +972,7 @@ axes: display_name: jruby-9.4 variables: RVM_RUBY: "jruby-9.4" - + - id: "os" display_name: OS values: @@ -982,6 +982,9 @@ axes: - id: ubuntu2204 display_name: "Ubuntu 22.04" run_on: ubuntu2204-small + - id: ubuntu2004 + display_name: "Ubuntu 20.04" + run_on: ubuntu2004-small - id: rhel8 display_name: "RHEL 8" run_on: rhel80-small @@ -1530,7 +1533,7 @@ buildvariants: # (depending on server version and what's available), we can bump this to # the latest stable db version. mongodb-version: "5.3" - os: rhel8 + os: ubuntu2004 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" tasks: - name: "test-aws-auth" diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index e6f101e8bd..882c38b730 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -186,7 +186,7 @@ axes: display_name: jruby-9.4 variables: RVM_RUBY: "jruby-9.4" - + - id: "os" display_name: OS values: @@ -196,6 +196,9 @@ axes: - id: ubuntu2204 display_name: "Ubuntu 22.04" run_on: ubuntu2204-small + - id: ubuntu2004 + display_name: "Ubuntu 20.04" + run_on: ubuntu2004-small - id: rhel8 display_name: "RHEL 8" run_on: rhel80-small diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index aa7f68c59d..dbf9c1788c 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -376,7 +376,7 @@ buildvariants: # (depending on server version and what's available), we can bump this to # the latest stable db version. mongodb-version: <%= latest_5x_mdb %> - os: rhel8 + os: ubuntu2004 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" tasks: - name: "test-aws-auth" diff --git a/spec/shared b/spec/shared index ce1f8945bd..49d0076b1d 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit ce1f8945bd61e614ca4cc7a24d95d16071c46a59 +Subproject commit 49d0076b1d22444c38ae82b4e35c476904576862 From 80c567d09e0a5a69afaddbe2c8beb57e1ccd211d Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 31 Oct 2023 17:01:33 +0100 Subject: [PATCH 02/14] Use the new task definition --- .evergreen/config.yml | 2 +- .evergreen/config/common.yml.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b9aab5ba4b..06340d9a5f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -179,7 +179,7 @@ functions: IAM_AUTH_ECS_SECURITY_GROUP="${iam_auth_ecs_security_group}" IAM_AUTH_ECS_SUBNET_A="${iam_auth_ecs_subnet_a}" IAM_AUTH_ECS_SUBNET_B="${iam_auth_ecs_subnet_b}" - IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition}" + IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition_ubuntu2004}" IAM_WEB_IDENTITY_ISSUER="${iam_web_identity_issuer}" IAM_WEB_IDENTITY_JWKS_URI="${iam_web_identity_jwks_uri}" diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 90e8def128..67d72dc942 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -176,7 +176,7 @@ functions: IAM_AUTH_ECS_SECURITY_GROUP="${iam_auth_ecs_security_group}" IAM_AUTH_ECS_SUBNET_A="${iam_auth_ecs_subnet_a}" IAM_AUTH_ECS_SUBNET_B="${iam_auth_ecs_subnet_b}" - IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition}" + IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition_ubuntu2004}" IAM_WEB_IDENTITY_ISSUER="${iam_web_identity_issuer}" IAM_WEB_IDENTITY_JWKS_URI="${iam_web_identity_jwks_uri}" From 9d59751e5fcfd77a56ed0cc7f9f6d9a05115c371 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 10:11:28 +0100 Subject: [PATCH 03/14] Fix package versions --- .evergreen/provision-local | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/provision-local b/.evergreen/provision-local index 3ad2654b4b..1ce14d240c 100755 --- a/.evergreen/provision-local +++ b/.evergreen/provision-local @@ -27,5 +27,5 @@ sudo env DEBIAN_FRONTEND=noninteractive \ # Need binutils for `strings` utility per # https://aws.amazon.com/premiumsupport/knowledge-center/ecs-iam-task-roles-config-errors/ sudo env DEBIAN_FRONTEND=noninteractive \ - apt-get install -y libsnmp30 libyaml-0-2 gcc make git lsb-release \ - krb5-user bzip2 libgmp-dev python-pip python2.7-dev binutils + apt-get install -y libsnmp35 libyaml-0-2 gcc make git lsb-release \ + krb5-user bzip2 libgmp-dev python3-pip python2.7-dev binutils From 6ba7fdeb11f9eff451f09c3f82b5fec7ba7a95d7 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 11:29:43 +0100 Subject: [PATCH 04/14] Bump mrss dep --- spec/shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/shared b/spec/shared index 49d0076b1d..0f0d1a2fea 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit 49d0076b1d22444c38ae82b4e35c476904576862 +Subproject commit 0f0d1a2fea50e2dfd1c0a87d79e5da9207e88084 From 8f152fb4158eddd1684c0b520efd938955400e68 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 12:19:58 +0100 Subject: [PATCH 05/14] Use new script for authawsvenv --- .evergreen/run-tests-aws-auth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index 1813400700..97d8a8a410 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -97,7 +97,7 @@ case "$AUTH" in aws-web-identity) cd `dirname "$0"`/auth_aws - . ./activate_venv.sh + . ./activate-authawsvenv.sh export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" python -u lib/aws_unassign_instance_profile.py From 60d99fac875f41aada6c8bd12c1bcee8e5ae9b50 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 12:28:45 +0100 Subject: [PATCH 06/14] 3333 --- .evergreen/run-tests-aws-auth.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index 97d8a8a410..da55b1c838 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -4,6 +4,10 @@ set -e # IMPORTANT: Don't set trace (-x) to avoid secrets showing up in the logs. set +x +MRSS_ROOT=`dirname "$0"`/../spec/shared + +. $MRSS_ROOT/shlib/distro.sh +. $MRSS_ROOT/shlib/set_env.sh . `dirname "$0"`/functions.sh # When running in Evergreen, credentials are written to this file. From a244f4e74f9d720acfc83063566c2352b9d9b8ba Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 13:48:43 +0100 Subject: [PATCH 07/14] 3333 --- .evergreen/run-tests-aws-auth.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index da55b1c838..b20cf06082 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -10,6 +10,15 @@ MRSS_ROOT=`dirname "$0"`/../spec/shared . $MRSS_ROOT/shlib/set_env.sh . `dirname "$0"`/functions.sh +arch=`host_distro` + +show_local_instructions + +set_home +set_env_vars +set_env_python +set_env_ruby + # When running in Evergreen, credentials are written to this file. # In Docker they are already in the environment and the file does not exist. if test -f .env.private; then From 081c9a31922612444a2d6aa4f1d051119bade1bb Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 13:56:37 +0100 Subject: [PATCH 08/14] 3333 --- .evergreen/run-tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a2203eb79a..059937bc94 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -34,8 +34,6 @@ MRSS_ROOT=`dirname "$0"`/../spec/shared arch=`host_distro` -show_local_instructions - set_home set_env_vars set_env_python From 6fdb0c5d12e0351f84feb29652cfd39f6de5575d Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 14:03:58 +0100 Subject: [PATCH 09/14] Revert "3333" This reverts commit 081c9a31922612444a2d6aa4f1d051119bade1bb. --- .evergreen/run-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 059937bc94..a2203eb79a 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -34,6 +34,8 @@ MRSS_ROOT=`dirname "$0"`/../spec/shared arch=`host_distro` +show_local_instructions + set_home set_env_vars set_env_python From 327e12cbe045156bcda33f260701b8363ac6544e Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 14:05:35 +0100 Subject: [PATCH 10/14] 3333 --- .evergreen/run-tests-aws-auth.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index b20cf06082..3ee206b866 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -12,8 +12,6 @@ MRSS_ROOT=`dirname "$0"`/../spec/shared arch=`host_distro` -show_local_instructions - set_home set_env_vars set_env_python From 35c1db050860fa68dc4ac65b3322dbda92898077 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 14:16:51 +0100 Subject: [PATCH 11/14] Try fixing web identity spec --- .evergreen/run-tests-aws-auth.sh | 49 +++++++++++++------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index 3ee206b866..fd73f4ad18 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -4,19 +4,8 @@ set -e # IMPORTANT: Don't set trace (-x) to avoid secrets showing up in the logs. set +x -MRSS_ROOT=`dirname "$0"`/../spec/shared - -. $MRSS_ROOT/shlib/distro.sh -. $MRSS_ROOT/shlib/set_env.sh . `dirname "$0"`/functions.sh -arch=`host_distro` - -set_home -set_env_vars -set_env_python -set_env_ruby - # When running in Evergreen, credentials are written to this file. # In Docker they are already in the environment and the file does not exist. if test -f .env.private; then @@ -106,25 +95,25 @@ case "$AUTH" in ;; aws-web-identity) - cd `dirname "$0"`/auth_aws - - . ./activate-authawsvenv.sh - export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" - export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" - python -u lib/aws_unassign_instance_profile.py - unset AWS_ACCESS_KEY_ID - unset AWS_SECRET_ACCESS_KEY - - export IDP_ISSUER="`get_var IAM_WEB_IDENTITY_ISSUER`" - export IDP_JWKS_URI="`get_var IAM_WEB_IDENTITY_JWKS_URI`" - export IDP_RSA_KEY="`get_var IAM_WEB_IDENTITY_RSA_KEY`" - export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" - python -u lib/aws_handle_oidc_creds.py token - unset IDP_ISSUER - unset IDP_JWKS_URI - unset IDP_RSA_KEY - - cd - + # cd `dirname "$0"`/auth_aws + # + # . ./activate-authawsvenv.sh + # export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" + # export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" + # python -u lib/aws_unassign_instance_profile.py + # unset AWS_ACCESS_KEY_ID + # unset AWS_SECRET_ACCESS_KEY + # + # export IDP_ISSUER="`get_var IAM_WEB_IDENTITY_ISSUER`" + # export IDP_JWKS_URI="`get_var IAM_WEB_IDENTITY_JWKS_URI`" + # export IDP_RSA_KEY="`get_var IAM_WEB_IDENTITY_RSA_KEY`" + # export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" + # python -u lib/aws_handle_oidc_creds.py token + # unset IDP_ISSUER + # unset IDP_JWKS_URI + # unset IDP_RSA_KEY + # + # cd - export MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" export MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" From d5bdf60624ae2fa3968e236a2579fbe2d83049d7 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 14:42:01 +0100 Subject: [PATCH 12/14] 3333 --- .evergreen/run-tests-aws-auth.sh | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index fd73f4ad18..e8c2fea7ba 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -95,25 +95,31 @@ case "$AUTH" in ;; aws-web-identity) - # cd `dirname "$0"`/auth_aws - # - # . ./activate-authawsvenv.sh - # export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" - # export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" - # python -u lib/aws_unassign_instance_profile.py - # unset AWS_ACCESS_KEY_ID - # unset AWS_SECRET_ACCESS_KEY - # - # export IDP_ISSUER="`get_var IAM_WEB_IDENTITY_ISSUER`" - # export IDP_JWKS_URI="`get_var IAM_WEB_IDENTITY_JWKS_URI`" - # export IDP_RSA_KEY="`get_var IAM_WEB_IDENTITY_RSA_KEY`" - # export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" - # python -u lib/aws_handle_oidc_creds.py token - # unset IDP_ISSUER - # unset IDP_JWKS_URI - # unset IDP_RSA_KEY - # - # cd - + cd `dirname "$0"`/auth_aws + + echo "Activating virtual environment 'authawsvenv'..." + . ./activate-authawsvenv.sh + echo "Virtual environment 'authawsvenv' activated." + export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" + export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" + echo "Unassigning instance profile..." + python -u lib/aws_unassign_instance_profile.py + echo "Instance profile unassigned." + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + + export IDP_ISSUER="`get_var IAM_WEB_IDENTITY_ISSUER`" + export IDP_JWKS_URI="`get_var IAM_WEB_IDENTITY_JWKS_URI`" + export IDP_RSA_KEY="`get_var IAM_WEB_IDENTITY_RSA_KEY`" + export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" + echo "Obtaining token..." + python -u lib/aws_handle_oidc_creds.py token + echo "Token obtained." + unset IDP_ISSUER + unset IDP_JWKS_URI + unset IDP_RSA_KEY + + cd - export MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" export MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" From fc980216068e7b3211b87d80b39fda972e532b1a Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 14:56:15 +0100 Subject: [PATCH 13/14] 3333 --- .evergreen/run-tests-aws-auth.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index e8c2fea7ba..c9649e4a64 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -99,12 +99,10 @@ case "$AUTH" in echo "Activating virtual environment 'authawsvenv'..." . ./activate-authawsvenv.sh - echo "Virtual environment 'authawsvenv' activated." export AWS_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" export AWS_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" echo "Unassigning instance profile..." python -u lib/aws_unassign_instance_profile.py - echo "Instance profile unassigned." unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY @@ -112,13 +110,12 @@ case "$AUTH" in export IDP_JWKS_URI="`get_var IAM_WEB_IDENTITY_JWKS_URI`" export IDP_RSA_KEY="`get_var IAM_WEB_IDENTITY_RSA_KEY`" export AWS_WEB_IDENTITY_TOKEN_FILE="`get_var IAM_WEB_IDENTITY_TOKEN_FILE`" - echo "Obtaining token..." python -u lib/aws_handle_oidc_creds.py token - echo "Token obtained." unset IDP_ISSUER unset IDP_JWKS_URI unset IDP_RSA_KEY + deactivate cd - export MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID="`get_var IAM_AUTH_EC2_INSTANCE_ACCOUNT`" export MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY="`get_var IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY`" From c44185ce0f26befb4777358a1023b814fc41926c Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 2 Nov 2023 15:11:49 +0100 Subject: [PATCH 14/14] Pull latest shared --- spec/shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/shared b/spec/shared index 0f0d1a2fea..bd968a969a 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit 0f0d1a2fea50e2dfd1c0a87d79e5da9207e88084 +Subproject commit bd968a969aed7ae4d579855f2ce5b3e3201444a4