DRIVERS-3158 Use AWS ECR pull through cache for Docker (#651)

Author: blink1073

.evergreen/auth_oidc/start_local_server.sh

@@ -27,9 +27,8 @@ EOF
 
 ENTRYPOINT=${ENTRYPOINT:-/root/docker_entry.sh}
 USE_TTY=""
-AWS_PROFILE=${AWS_PROFILE:-""}
 
-if [ -z "$AWS_PROFILE" ]; then
+if [ -z "${AWS_PROFILE:-}" ]; then
     if [[ -z "${AWS_SESSION_TOKEN:-}" ||  -z "${AWS_ACCESS_KEY_ID:-}" || -z "${AWS_SECRET_ACCESS_KEY:-}" ]]; then
         echo "Please set AWS_PROFILE or set AWS credentials environment variables" 1>&2
        exit 1
@@ -55,6 +54,11 @@ if [ -n "${DOCKER_COMMAND:-}" ]; then
     DOCKER=$DOCKER_COMMAND
 fi
 
+# Log in to docker if running on CI.
+if [ -n "${CI:-}" ]; then
+    bash $SCRIPT_DIR/../docker/setup.sh
+fi
+
 # Build from the root directory so we can include files.
 pushd $DRIVERS_TOOLS
 PLATFORM="--platform linux/amd64"

.evergreen/config.yml

@@ -306,6 +306,9 @@ functions:
           - SSL
           - STORAGE_ENGINE
           - TOPOLOGY
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
         args:
           - ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
           - -v
@@ -349,6 +352,7 @@ functions:
         binary: bash
         env:
           ENTRYPOINT: /root/test-entrypoint.sh
+        include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
         args:
           - ${DRIVERS_TOOLS}/.evergreen/docker/run-server.sh
     - command: subprocess.exec

.evergreen/docker/README.md

@@ -7,7 +7,25 @@ for your Driver that shares the binary files and communicates with this containe
 
 You will need podman (or Docker) installed and running locally.
 
-# Run Local Server
+## Log in to Docker
+
+Run the `setup.sh` script to log into docker.  This is needed when using the ECR pull-through
+cache that DevProd maintains.  This script can be run after assuming the Drivers test secrets role.  For example:
+
+```yaml
+"docker login":
+  - command: ec2.assume_role
+    params:
+    role_arn: ${drivers_test_secrets_role}
+  - command: subprocess.exec
+    type: test
+    params:
+    binary: bash
+    include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
+    args: ["${DRIVERS_TOOLS}/.evergreen/docker/setup.sh"]
+```
+
+## Run Local Server
 
 To run a local server, change to this directory and run:
 

.evergreen/docker/activate-dockerenv.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# activate-dockervenv.sh
+#
+# Usage:
+#   . ./activate-dockervenv.sh
+#
+# This file creates and/or activates the dockervenv virtual environment in the
+# current working directory. This file must be invoked from within the
+# .evergreen/ocsp directory in the Drivers Evergreen Tools repository.
+#
+# If a dockervenv virtual environment already exists, it will be activated and
+# no further action will be taken. If a dockervenv virtual environment must be
+# created, required packages will also be installed.
+
+# If an error occurs during creation, activation, or installation of packages,
+# the dockervenv virtual environment will be deactivated and activate_dockervenv
+# will return a non-zero value.
+
+if [ -z "$BASH" ]; then
+  echo "activate-dockervenv.sh must be run in a Bash shell!" 1>&2
+  return 1
+fi
+
+# Automatically invoked by activate-dockervenv.sh.
+activate_dockervenv() {
+  # shellcheck source=.evergreen/venv-utils.sh
+  . ../venv-utils.sh || return
+
+  if [[ -d dockervenv ]]; then
+    venvactivate dockervenv || return
+  else
+    # shellcheck source=.evergreen/find-python3.sh
+    . ../find-python3.sh || return
+    PYTHON=$(ensure_python3) || return
+
+    echo "Creating virtual environment 'dockervenv'..."
+    venvcreate "${PYTHON:?}" dockervenv || return
+
+    python -m pip install -q -r requirements.txt || {
+      local -r ret="$?"
+      deactivate || return 1 # Deactivation should never fail!
+      return "$ret"
+    }
+    echo "Creating virtual environment 'dockervenv'... done."
+  fi
+}
+
+activate_dockervenv

.evergreen/docker/login.py

@@ -0,0 +1,55 @@
+"""
+Python script to log in to the Dev Prod ECR registry.
+"""
+
+import base64
+import os
+import shlex
+import shutil
+import subprocess
+
+import boto3
+
+registry = os.environ["ECR_REGISTRY"]
+account = registry.split(".")[0]
+if "CI" in os.environ:
+    sts_client = boto3.client("sts", region_name=os.environ["AWS_REGION"])
+    resp = sts_client.assume_role(
+        RoleArn=os.environ["AWS_ROLE_ARN"],
+        RoleSessionName=f"{account}-test",
+        ExternalId=os.environ["AWS_EXTERNAL_ID"],
+    )
+    creds = resp["Credentials"]
+    sts_client.close()
+else:
+    creds = dict(AccessKeyId=None, SecretAccessKey=None, SessionToken=None)
+
+ecr_client = boto3.client(
+    "ecr",
+    aws_access_key_id=creds["AccessKeyId"],
+    aws_secret_access_key=creds["SecretAccessKey"],
+    aws_session_token=creds["SessionToken"],
+    region_name=os.environ["AWS_REGION"],
+)
+resp = ecr_client.get_authorization_token(registryIds=[account])
+ecr_client.close()
+
+token = resp["authorizationData"][0]["authorizationToken"]
+_, _, token = base64.b64decode(token).partition(b":")
+
+docker = shutil.which("docker") or shutil.which("podman")
+if "podman" in docker:
+    docker = f"sudo {docker}"
+
+cmd = f"{docker} login --username AWS --password-stdin {registry}"
+proc = subprocess.Popen(
+    shlex.split(cmd),
+    stdout=subprocess.PIPE,
+    stdin=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+)
+stdout, stderr = proc.communicate(token)
+if stdout:
+    print(stdout.decode("utf-8").strip())
+if stderr:
+    print(stderr.decode("utf-8").strip())

.evergreen/docker/requirements.txt

@@ -0,0 +1,2 @@
+boto3~=1.35.0
+docker~=7.1.0

.evergreen/docker/run-server.sh

@@ -14,6 +14,11 @@ PLATFORM=${DOCKER_PLATFORM:-}
 ARCH=${ARCH:-}
 # e.g. --platform linux/amd64
 
+# Log in to docker if running on CI.
+if [ -n "${CI:-}" ]; then
+    bash $SCRIPT_DIR/setup.sh
+fi
+
 if [[ -z $PLATFORM && -n $ARCH ]]; then
     PLATFORM="--platform linux/$ARCH"
 fi

.evergreen/docker/setup.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+pushd $SCRIPT_DIR
+
+# Source secrets from the vault.
+if [ ! -f secrets-export.sh ]; then
+  . $SCRIPT_DIR/../secrets_handling/setup-secrets.sh drivers/docker
+fi
+source secrets-export.sh
+
+# Python script that uses boto3 to assume the role and get the login password, then passes it to docker login
+. ./activate-dockerenv.sh
+python login.py

.evergreen/docker/ubuntu20.04/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:20.04
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
   apt-get -qq update && apt-get -qq -y install --no-install-recommends \

.evergreen/orchestration/drivers_orchestration.py

@@ -246,9 +246,15 @@ def run_command(cmd: str, exit_on_error=True, **kwargs):
 
 def start_atlas(opts):
     mo_home = Path(opts.mongo_orchestration_home)
-    image = f"docker.io/mongodb/mongodb-atlas-local:{opts.version}"
+    image = f"mongodb/mongodb-atlas-local:{opts.version}"
     docker = get_docker_cmd()
     stop(opts)
+    # If we're on evergreen, we need to log into docker and use the pull-through cache.
+    if "CI" in os.environ and "GITHUB_ACTION" not in os.environ:
+        LOGGER.info("Logging in to docker...")
+        run_command("bash setup.sh", cwd=EVG_PATH / "docker")
+        LOGGER.info("Logging in to ECR... done.")
+        image = f"901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/{image}"
     cmd = f"{docker} run --rm -d --name mongodb_atlas_local -p 27017:27017"
     if opts.auth:
         cmd += " -e MONGODB_INITDB_ROOT_USERNAME=bob"

.evergreen/secrets_handling/setup_secrets.py

@@ -15,14 +15,15 @@
 AWS_ROLE_ARN = "arn:aws:iam::857654397073:role/drivers-test-secrets-role"
 
 
-def get_secrets(vaults, region, profile):
+def get_secrets(vaults, region, profile, assume_role=False):
     """Get the driver secret values."""
     # Handle local credentials.
     profile = profile or os.environ.get("AWS_PROFILE")
     session = boto3.Session(profile_name=profile)
     creds = None
     kwargs = dict(region_name=region)
-    if "AWS_ACCESS_KEY_ID" not in os.environ and not profile:
+    if assume_role or ("AWS_ACCESS_KEY_ID" not in os.environ and not profile):
+        assume_role = True
         client = session.client(service_name="sts", **kwargs)
         try:
             # This will only fail locally.
@@ -49,12 +50,16 @@ def get_secrets(vaults, region, profile):
         for vault in vaults:
             secret = client.get_secret_value(SecretId=vault)["SecretString"]
             secrets.append(secret)
-    except botocore.exceptions.BotoCoreError as e:
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        # Try one more time with assume_role.
+        if not assume_role:
+            return get_secrets(vaults, region, profile, True)
         # For a list of exceptions thrown, see
         # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
         print(f"\nERROR: {e}\n")
         sys.exit(1)
     except Exception as e:
+        print("HERE I am also, I caught the exception!", assume_role)
         raise e
 
     # Decrypts secret using the associated KMS key.

.github/workflows/tests.yml

@@ -20,6 +20,7 @@ jobs:
   tests:
     name: "Tests"
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
 
     strategy:
       fail-fast: false