Move Jenkins jobs to Buildkite

.buildkite/Dockerfile

@@ -0,0 +1,13 @@
+ARG PYTHON_VERSION=3.8
+FROM python:${PYTHON_VERSION}
+
+WORKDIR /code/elasticsearch-py
+
+COPY dev-requirements.txt .
+RUN python -m pip install \
+    -U --no-cache-dir \
+    --disable-pip-version-check \
+    -r dev-requirements.txt
+
+COPY . .
+RUN python -m pip install -U -e .

.ci/functions/cleanup.sh → .buildkite/functions/cleanup.sh

@@ -2,7 +2,7 @@
 #
 # Shared cleanup routines between different steps
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.0
 # - Initial version after refactor

.ci/functions/wait-for-container.sh → .buildkite/functions/wait-for-container.sh

@@ -2,7 +2,7 @@
 #
 # Exposes a routine scripts can call to wait for a container if that container set up a health command
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.1
 # - Initial version after refactor

.buildkite/pipeline.yml

@@ -1,3 +1,39 @@
+---
 steps:
-  - label: ":python: Greetings"
-    command: "echo 'Hello, world!'"
+  - label: ":python: :elasticsearch:"
+    agents:
+      provider: gcp
+    matrix:
+      setup:
+        stack_version:
+          - "8.11.0-SNAPSHOT"
+        test_suite:
+          - platinum
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        connection_class:
+          - urllib3
+          - requests
+    env:
+      STACK_VERSION: "{{ matrix.stack_version }}"
+      TEST_SUITE: "{{ matrix.test_suite }}"
+      PYTHON_VERSION: "{{ matrix.python }}"
+      PYTHON_CONNECTION_CLASS: "{{ matrix.connection_class }}"
+    command: ./.buildkite/run-tests
+    artifact_paths: "junit/*-junit.xml"
+  - wait: ~
+    continue_on_failure: true
+  - label: ":junit: Test results"
+    agents:
+      provider: "gcp"
+      image: family/core-ubuntu-2204
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "junit/*-junit.xml"
+          job-uuid-file-pattern: "(.*)-junit.xml"
+          fail-build-on-error: true
+          failure-format: file

.ci/run-elasticsearch.sh → .buildkite/run-elasticsearch.sh

@@ -25,12 +25,16 @@ script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
-echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
 cleanup_node $es_node_name
 
 master_node_name=${es_node_name}
 cluster_name=${moniker}${suffix}
 
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
 declare -a volumes
 environment=($(cat <<-END
   --env ELASTIC_PASSWORD=$elastic_password

.buildkite/run-repository.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Called by entry point `run-test` use this script to add your repository specific test commands
+# Once called Elasticsearch is up and running and the following parameters are available to this script
+
+# STACK_VERSION -- version e.g Major.Minor.Patch(-Prelease)
+# elasticsearch_container -- the docker moniker as a reference to know which docker image distribution is used
+# elasticsearch_url -- The url at which elasticsearch is reachable
+# network_name -- The docker network name
+# node_name -- The docker container name also used as Elasticsearch node name
+
+# When run in CI the test-matrix is used to define additional variables
+# TEST_SUITE -- either `oss` or `xpack`, defaults to `oss` in `run-tests`
+
+set -e
+
+echo -e "\033[34;1mINFO:\033[0m URL $elasticsearch_url\033[0m"
+echo -e "\033[34;1mINFO:\033[0m VERSION $STACK_VERSION\033[0m"
+echo -e "\033[34;1mINFO:\033[0m CONTAINER $elasticsearch_container\033[0m"
+echo -e "\033[34;1mINFO:\033[0m TEST_SUITE $TEST_SUITE\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION $PYTHON_VERSION\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS $PYTHON_CONNECTION_CLASS\033[0m"
+
+echo "--- :docker: :python: Build elastic/elasticsearch-py container"
+
+docker build \
+       --file .buildkite/Dockerfile \
+       --tag elastic/elasticsearch-py \
+       --build-arg "PYTHON_VERSION=$PYTHON_VERSION" \
+       .
+
+echo "--- :docker: :python: Run elastic/elasticsearch-py container"
+
+mkdir -p "$(pwd)/junit"
+docker run \
+  --network="$network_name" \
+  -e STACK_VERSION \
+  -e "ELASTICSEARCH_URL=$elasticsearch_url" \
+  -e TEST_SUITE \
+  -e PYTHON_CONNECTION_CLASS \
+  -v "$(pwd)/junit:/junit" \
+  --name elasticsearch-py \
+  --rm \
+  elastic/elasticsearch-py \
+  bash -c "nox -s test-$PYTHON_VERSION; [ -f ./junit/$BUILDKITE_JOB_ID-junit.xml ] && mv ./junit/$BUILDKITE_JOB_ID-junit.xml /junit || echo 'No JUnit artifact found'"

.ci/run-tests → .buildkite/run-tests

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
 # Version 1.1
-# - Moved to .ci folder and seperated out `run-repository.sh`
+# - Moved to .buildkite folder and separated out `run-repository.sh`
 # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
 
 # Default environment variables
@@ -11,20 +11,20 @@ export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
 
 script_path=$(dirname $(realpath -s $0))
-source $script_path/functions/imports.sh
+source "$script_path/functions/imports.sh"
 set -euo pipefail
 
-echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-DETACH=true bash .ci/run-elasticsearch.sh
+echo "--- :elasticsearch: Starting Elasticsearch $STACK_VERSION"
+DETACH=true bash .buildkite/run-elasticsearch.sh
 
 if [[ -n "$RUNSCRIPTS" ]]; then
   for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
-    echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-    CONTAINER_NAME=${RUNSCRIPT} \
+    echo "--- Running run-$RUNSCRIPT.sh"
+    CONTAINER_NAME="$RUNSCRIPT" \
       DETACH=true \
-      bash .ci/run-${RUNSCRIPT}.sh
+      bash ".buildkite/run-$RUNSCRIPT.sh"
   done
 fi
 
-echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-bash .ci/run-repository.sh
+echo "--- :python: Run tests"
+bash .buildkite/run-repository.sh

.ci/make.sh

@@ -114,7 +114,7 @@ echo -e "\033[34;1mINFO: building $product container\033[0m"
 
 docker build \
   --build-arg BUILDER_UID="$(id -u)" \
-  --file $repo/.ci/Dockerfile \
+  --file $repo/.buildkite/Dockerfile \
   --tag ${product} \
   .