Get run-tests script working

Pulls secrets in from all the necessary places, uses them securely, and
ensures serverless QA instance gets deleted even if the script fails.

Author: JoshMock

.buildkite/rest-tests.yaml

@@ -30,7 +30,7 @@ steps:
           - "3.10"
           - "3.11"
         connection_class:
-          - urrlib3
+          - urllib3
           - requests
     command: ./.buildkite/run-tests
     artifact_paths: "junit/*-junit.xml"

.buildkite/run-tests

@@ -7,18 +7,39 @@ export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
 
 set -euo pipefail
 
+# fetch cloud creds used by qaf
+CLOUD_ACCESS_KEY=$(vault read -field="$EC_ENV" secret/ci/elastic-elasticsearch-serverless-python/cloud-access)
+echo "{\"api_key\":{\"$EC_ENV\":\"$CLOUD_ACCESS_KEY\"}}" > "$(pwd)/cloud.json"
+
+# ensure serverless instance is deleted even if script errors
+cleanup() {
+  echo -e "--- :elasticsearch: Tear down serverless instance EC_PROJECT_NAME"
+  docker run --rm \
+    -e EC_REGISTER_BACKEND \
+    -e EC_ENV \
+    -e EC_REGION \
+    -e EC_PROJECT_NAME \
+    -e VAULT_TOKEN \
+    -v "$(pwd)/cloud.json:/root/.elastic/cloud.json" \
+    docker.elastic.co/employees/dolaru/qaf:latest \
+    bash -c 'qaf elastic-cloud projects delete'
+}
+trap cleanup EXIT
+
 echo -e "--- :elasticsearch: Start serverless instance"
+
 deployment=$(docker run --rm \
   -e EC_REGISTER_BACKEND \
   -e EC_ENV \
   -e EC_REGION \
   -e EC_PROJECT_NAME \
+  -e VAULT_TOKEN \
+  -v "$(pwd)/cloud.json:/root/.elastic/cloud.json" \
   docker.elastic.co/employees/dolaru/qaf:latest \
   bash -c 'qaf elastic-cloud projects create --project-type elasticsearch && \
   qaf elastic-cloud projects describe --as-json --show-credentials') \
 
-ES_API_KEY=$(echo "$deployment" | jq -r '.credentials.api_key')
-export ES_API_KEY
+ES_API_SECRET_KEY=$(echo "$deployment" | jq -r '.credentials.api_key')
 ELASTICSEARCH_URL=$(echo "$deployment" | jq -r '.elasticsearch.url')
 export ELASTICSEARCH_URL
 
@@ -39,23 +60,18 @@ docker build \
 
 echo -e "--- :docker: :python: Run integration tests for Python $PYTHON_VERSION"
 
+GITHUB_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch-serverless-python/github-token)
+export GITHUB_TOKEN
+
 docker run \
   -e ELASTICSEARCH_URL \
-  -e ES_API_KEY \
+  -e "ES_API_KEY=$ES_API_SECRET_KEY" \
   -e PYTHON_CONNECTION_CLASS \
   -e STACK_VERSION \
   -e TEST_SUITE \
+  -e GITHUB_TOKEN \
   --name elasticsearch-serverless-python-tests \
   --volume "$(pwd)/junit:/code/elasticsearch-serverless-python/junit" \
   --rm \
   elasticsearch-serverless-python \
   nox -s "test-$PYTHON_VERSION"
-
-echo -e "--- :elasticsearch: Tear down serverless instance"
-deployment=$(docker run --rm \
-  -e EC_REGISTER_BACKEND \
-  -e EC_ENV \
-  -e EC_REGION \
-  -e EC_PROJECT_NAME \
-  docker.elastic.co/employees/dolaru/qaf:latest \
-  bash -c 'qaf elastic-cloud projects delete') \