Skip to content

Commit 8816f94

Browse files
picandocodigopicandocodigo
committed
[CI] Updates Buildkite configuration
1 parent 08da486 commit 8816f94

File tree

13 files changed

+529
-2
lines changed

13 files changed

+529
-2
lines changed

‎.buildkite/Dockerfile

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
ARG PYTHON_VERSION=3.8
2+
FROM python:${PYTHON_VERSION}
3+
4+
# Default UID/GID to 1000
5+
# it can be overridden at build time
6+
ARG BUILDER_UID=1000
7+
ARG BUILDER_GID=1000
8+
ENV BUILDER_USER elastic
9+
ENV BUILDER_GROUP elastic
10+
ENV PATH="${PATH}:/var/lib/elastic/.local/bin"
11+
12+
# Create user
13+
RUN groupadd --system -g ${BUILDER_GID} ${BUILDER_GROUP} \
14+
&& useradd --system --shell /bin/bash -u ${BUILDER_UID} -g ${BUILDER_GROUP} -d /var/lib/elastic -m elastic 1>/dev/null 2>/dev/null \
15+
&& mkdir -p /code/elasticsearch-py && mkdir /code/elasticsearch-py/build \
16+
&& chown -R ${BUILDER_USER}:${BUILDER_GROUP} /code/
17+
COPY --chown=$BUILDER_USER:$BUILDER_GROUP . .
18+
WORKDIR /code/elasticsearch-py
19+
USER ${BUILDER_USER}:${BUILDER_GROUP}
20+
COPY dev-requirements.txt .
21+
RUN python -m pip install \
22+
-U --no-cache-dir \
23+
--disable-pip-version-check \
24+
nox -rdev-requirements.txt
25+
COPY --chown=$BUILDER_USER:$BUILDER_GROUP . .
26+
RUN python -m pip install -U -e .

‎.buildkite/certs/ca.crt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDSjCCAjKgAwIBAgIVAJQLm8V2LcaCTHUcoIfO+KL63nG3MA0GCSqGSIb3DQEB
3+
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
4+
ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1N1oXDTIzMDIyNTA1NTA1N1owNDEyMDAG
5+
A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
6+
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYyajkPvGtUOE5M1OowQfB
7+
kWVrWjo1+LIxzgCeRHp0YztLtdVJ0sk2xoSrt2uZpxcPepdyOseLTjFJex1D2yCR
8+
AEniIqcFif4G72nDih2LlbhpUe/+/MTryj8ZTkFTzI+eMmbQi5FFMaH+kwufmdt/
9+
5/w8YazO18SxxJUlzMqzfNUrhM8vvvVdxgboU7PWhk28wZHCMHQovomHmzclhRpF
10+
N0FMktA98vHHeRjH19P7rNhifSd7hZzoH3H148HVAKoPgqnZ6vW2O2YfAWOP6ulq
11+
cyszr57p8fS9B2wSdlWW7nVHU1JuKcYD67CxbBS23BeGFgCj4tiNrmxO8S5Yf85v
12+
AgMBAAGjUzBRMB0GA1UdDgQWBBSWAlip9eoPmnG4p4OFZeOUBlAbNDAfBgNVHSME
13+
GDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
14+
SIb3DQEBCwUAA4IBAQA19qqrMTWl7YyId+LR/QIHDrP4jfxmrEELrAL58q5Epc1k
15+
XxZLzOBSXoBfBrPdv+3XklWqXrZjKWfdkux0Xmjnl4qul+srrZDLJVZG3I7IrITh
16+
AmQUmL9MuPiMnAcxoGZp1xpijtW8Qmd2qnambbljWfkuVaa4hcVRfrAX6TciIQ21
17+
bS5aeLGrPqR14h30YzDp0RMmTujEa1o6ExN0+RSTkE9m89Q6WdM69az8JW7YkWqm
18+
I+UCG3TcLd3TXmN1zNQkq4y2ObDK4Sxy/2p6yFPI1Fds5w/zLfBOvvPQY61vEqs8
19+
SCCcQIe7f6NDpIRIBlty1C9IaEHj7edyHjF6rtYb
20+
-----END CERTIFICATE-----

‎.buildkite/certs/ca.pem

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDSjCCAjKgAwIBAgIVAJQLm8V2LcaCTHUcoIfO+KL63nG3MA0GCSqGSIb3DQEB
3+
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
4+
ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1N1oXDTIzMDIyNTA1NTA1N1owNDEyMDAG
5+
A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
6+
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYyajkPvGtUOE5M1OowQfB
7+
kWVrWjo1+LIxzgCeRHp0YztLtdVJ0sk2xoSrt2uZpxcPepdyOseLTjFJex1D2yCR
8+
AEniIqcFif4G72nDih2LlbhpUe/+/MTryj8ZTkFTzI+eMmbQi5FFMaH+kwufmdt/
9+
5/w8YazO18SxxJUlzMqzfNUrhM8vvvVdxgboU7PWhk28wZHCMHQovomHmzclhRpF
10+
N0FMktA98vHHeRjH19P7rNhifSd7hZzoH3H148HVAKoPgqnZ6vW2O2YfAWOP6ulq
11+
cyszr57p8fS9B2wSdlWW7nVHU1JuKcYD67CxbBS23BeGFgCj4tiNrmxO8S5Yf85v
12+
AgMBAAGjUzBRMB0GA1UdDgQWBBSWAlip9eoPmnG4p4OFZeOUBlAbNDAfBgNVHSME
13+
GDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
14+
SIb3DQEBCwUAA4IBAQA19qqrMTWl7YyId+LR/QIHDrP4jfxmrEELrAL58q5Epc1k
15+
XxZLzOBSXoBfBrPdv+3XklWqXrZjKWfdkux0Xmjnl4qul+srrZDLJVZG3I7IrITh
16+
AmQUmL9MuPiMnAcxoGZp1xpijtW8Qmd2qnambbljWfkuVaa4hcVRfrAX6TciIQ21
17+
bS5aeLGrPqR14h30YzDp0RMmTujEa1o6ExN0+RSTkE9m89Q6WdM69az8JW7YkWqm
18+
I+UCG3TcLd3TXmN1zNQkq4y2ObDK4Sxy/2p6yFPI1Fds5w/zLfBOvvPQY61vEqs8
19+
SCCcQIe7f6NDpIRIBlty1C9IaEHj7edyHjF6rtYb
20+
-----END CERTIFICATE-----

‎.buildkite/certs/testnode.crt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDYjCCAkqgAwIBAgIVAIZQH0fe5U+bGQ6m1JUBO/AQkQ/9MA0GCSqGSIb3DQEB
3+
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
4+
ZXJhdGVkIENBMB4XDTIwMDMyNzE5MTcxMVoXDTIzMDMyNzE5MTcxMVowEzERMA8G
5+
A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB
6+
fco1t1+sE1gTwTVGcXKZqJTP2GjMHM0cfJE5KKfwC5B+pHADRT6FZxvepgKjEBDt
7+
CK+2Rmotyeb15XXMSKguNhyT+2PuKvT5r05L7P91XRYXrwxG2swJPtct7A87xdFa
8+
Ek+YRpqGGmTaux2jOELMiAmqEzoj6w/xFq+LF4SolTW4wOL2eLFkEFHBX2oCwU5T
9+
Q+B+7E9zL45nFWlkeRGJ+ZQTnRNZ/1r4N9A9Gtj4x/H1/y4inWndikdxAb5QiEYJ
10+
T+vbQWzHYWjz13ttHJsz+6T8rvA1jK+buHgVh4K8lV13X9k54soBqHB8va7/KIJP
11+
g8gvd6vusEI7Bmfl1as7AgMBAAGjgYswgYgwHQYDVR0OBBYEFKnnpvuVYwtFSUis
12+
WwN9OHLyExzJMB8GA1UdIwQYMBaAFJYCWKn16g+acbing4Vl45QGUBs0MDsGA1Ud
13+
EQQ0MDKCCWxvY2FsaG9zdIIIaW5zdGFuY2WHBH8AAAGHEAAAAAAAAAAAAAAAAAAA
14+
AAGCA2VzMTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAPNsIoD4GBrTgR
15+
jfvBuHS6eU16P95m16O8Mdpr4SMQgWLQUhs8aoVgfwpg2TkbCWxOe6khJOyNm7bf
16+
fW4aFQ/OHcQV4Czz3c7eOHTWSyMlCOv+nRXd4giJZ5TOHw1zKGmKXOIvhvE6RfdF
17+
uBBfrusk164H4iykm0Bbr/wo4d6wuebp3ZYLPw5zV0D08rsaR+3VJ9VxWuFpdm/r
18+
2onYOohyuX9DRjAczasC+CRRQN4eHJlRfSQB8WfTKw3EloRJJDAg6SJyGiAJ++BF
19+
hnqfNcEyKes2AWagFF9aTbEJMrzMhH+YB5F+S/PWvMUlFzcoocVKqc4pIrjKUNWO
20+
6nbTxeAB
21+
-----END CERTIFICATE-----

‎.buildkite/certs/testnode.key

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEowIBAAKCAQEAwX3KNbdfrBNYE8E1RnFymaiUz9hozBzNHHyROSin8AuQfqRw
3+
A0U+hWcb3qYCoxAQ7QivtkZqLcnm9eV1zEioLjYck/tj7ir0+a9OS+z/dV0WF68M
4+
RtrMCT7XLewPO8XRWhJPmEaahhpk2rsdozhCzIgJqhM6I+sP8RavixeEqJU1uMDi
5+
9nixZBBRwV9qAsFOU0PgfuxPcy+OZxVpZHkRifmUE50TWf9a+DfQPRrY+Mfx9f8u
6+
Ip1p3YpHcQG+UIhGCU/r20Fsx2Fo89d7bRybM/uk/K7wNYyvm7h4FYeCvJVdd1/Z
7+
OeLKAahwfL2u/yiCT4PIL3er7rBCOwZn5dWrOwIDAQABAoIBAFcm4ICnculf4Sks
8+
umFbUiISA81GjZV6V4zAMu1K+bGuk8vnJyjh9JJD6hK0NbXa07TgV7zDJKoxKd2S
9+
GCgGhfIin2asMcuh/6vDIYIjYsErR3stdlsnzAVSD7v4ergSlwR6AO32xz0mAE1h
10+
QK029yeHEstPU72/7/NIo5MD6dXAbut1MzgijZD8RQo1z21D6qmLcPTVTfkn7a3W
11+
MY3y7XUIkA1TOyIRsH3k6F6NBWkvtXbwOUeLCJ14EvS8T9BqhIhPDZv8mQTRLDOD
12+
tQRyC4Cnw+UhYmnMFJhj6N2jpTBv/AdoKcRC56uBJyPW+dxj6i4e7n3pQuxqRvpI
13+
LLJJsskCgYEA4QQxzuJizLKV75rE+Qxg0Ej0Gid1aj3H5eeTZOUhm9KC8KDfPdpk
14+
msKaNzJq/VDcqHPluGS1jYZVgZlal1nk5xKBcbQ4n297VPVd+sLtlf0bj4atlDUO
15+
+iOVo0H7k5yWvj+TzVRlc5zjDLcnQh8i+22o3+65hIrb2zpzg/cCZJ8CgYEA3CJX
16+
bjmWPQ0uZVIa8Wz8cJFtKT9uVl7Z3/f6HjN9I0b/9MmVlNxQVAilVwhDkzR/UawG
17+
QeRFBJ6XWRwX0aoMq+O9VSNu/R2rtEMpIYt3LwbI3yw6GRoCdB5qeL820O+KX5Fl
18+
/z+ZNgrHgA1yKPVf+8ke2ZtLEqPHMN+BMuq8t+UCgYEAy0MfvzQPbbuw55WWcyb0
19+
WZJdNzcHwKX4ajzrj4vP9VOPRtD7eINMt+QsrMnVjei6u0yeahhHTIXZvc2K4Qeq
20+
V/YGinDzaUqqTU+synXFauUOPXO6XxQi6GC2rphPKsOcBFWoLSYc0vgYvgbA5uD7
21+
l8Yyc77RROKuwfWmHcJHHh8CgYBurGFSjGdJWHgr/oSHPqkIG0VLiJV7nQJjBPRd
22+
/Lr8YnTK6BJpHf7Q0Ov3frMirjEYqakXtaExel5TMbmT8q+eN8h3pnHlleY+oclr
23+
EQghv4J8GWs4NYhoQuZ6wH/ZuaTS+XHTS3FG51J3wcrUZtET8ICvHNE4lNjPbH8z
24+
TysENQKBgHER1RtDFdz+O7mlWibrHk8JDgcVdZV/pBF+9cb7r/orkH9RLAHDlsAO
25+
tuSVaQmm5eqgaAxMamBXSyw1lir07byemyuEDg0mJ1rNUGsAY8P+LWr579gvKMme
26+
5gvrJr99JkBTV3z+TiL7dZa52eW00Ijqg2qcbHGpq3kXWWkbd8Tn
27+
-----END RSA PRIVATE KEY-----

‎.buildkite/functions/cleanup.sh

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Shared cleanup routines between different steps
4+
#
5+
# Please source .ci/functions/imports.sh as a whole not just this file
6+
#
7+
# Version 1.0.0
8+
# - Initial version after refactor
9+
10+
function cleanup_volume {
11+
if [[ "$(docker volume ls -q -f name=$1)" ]]; then
12+
echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
13+
(docker volume rm "$1") || true
14+
fi
15+
}
16+
function container_running {
17+
if [[ "$(docker ps -q -f name=$1)" ]]; then
18+
return 0;
19+
else return 1;
20+
fi
21+
}
22+
function cleanup_node {
23+
if container_running "$1"; then
24+
echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
25+
(docker container rm --force --volumes "$1") || true
26+
fi
27+
if [[ -n "$1" ]]; then
28+
echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
29+
cleanup_volume "$1-${suffix}-data"
30+
fi
31+
}
32+
function cleanup_network {
33+
if [[ "$(docker network ls -q -f name=$1)" ]]; then
34+
echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
35+
(docker network rm "$1") || true
36+
fi
37+
}
38+
39+
function cleanup_trap {
40+
status=$?
41+
set +x
42+
if [[ "$DETACH" != "true" ]]; then
43+
echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
44+
cleanup_all_in_network "$1"
45+
fi
46+
# status is 0 or SIGINT
47+
if [[ "$status" == "0" || "$status" == "130" ]]; then
48+
echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
49+
exit 0
50+
else
51+
echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
52+
exit ${status}
53+
fi
54+
};
55+
function cleanup_all_in_network {
56+
57+
if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
58+
echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
59+
return 0
60+
fi
61+
containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
62+
while read -r container; do
63+
cleanup_node "$container"
64+
done <<< "$containers"
65+
cleanup_network $1
66+
echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
67+
};

‎.buildkite/functions/imports.sh

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Sets up all the common variables and imports relevant functions
4+
#
5+
# Version 1.0.1
6+
# - Initial version after refactor
7+
# - Validate STACK_VERSION asap
8+
9+
function require_stack_version() {
10+
if [[ -z $STACK_VERSION ]]; then
11+
echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
12+
exit 1
13+
fi
14+
}
15+
16+
require_stack_version
17+
18+
if [[ -z $es_node_name ]]; then
19+
# only set these once
20+
set -euo pipefail
21+
export TEST_SUITE=${TEST_SUITE-platinum}
22+
export RUNSCRIPTS=${RUNSCRIPTS-}
23+
export DETACH=${DETACH-false}
24+
export CLEANUP=${CLEANUP-false}
25+
26+
export es_node_name=instance
27+
export elastic_password=changeme
28+
export elasticsearch_image=elasticsearch
29+
export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
30+
if [[ $TEST_SUITE != "platinum" ]]; then
31+
export elasticsearch_url=http://${es_node_name}:9200
32+
fi
33+
export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
34+
export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
35+
36+
export suffix=rest-test
37+
export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
38+
export network_name=${moniker}${suffix}
39+
40+
export ssl_cert="${script_path}/certs/testnode.crt"
41+
export ssl_key="${script_path}/certs/testnode.key"
42+
export ssl_ca="${script_path}/certs/ca.crt"
43+
44+
fi
45+
46+
export script_path=$(dirname $(realpath -s $0))
47+
source $script_path/functions/cleanup.sh
48+
source $script_path/functions/wait-for-container.sh
49+
trap "cleanup_trap ${network_name}" EXIT
50+
51+
52+
if [[ "$CLEANUP" == "true" ]]; then
53+
cleanup_all_in_network $network_name
54+
exit 0
55+
fi
56+
57+
echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
58+
docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
59+

‎.buildkite/functions/wait-for-container.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Exposes a routine scripts can call to wait for a container if that container set up a health command
4+
#
5+
# Please source .ci/functions/imports.sh as a whole not just this file
6+
#
7+
# Version 1.0.1
8+
# - Initial version after refactor
9+
# - Make sure wait_for_contiainer is silent
10+
11+
function wait_for_container {
12+
set +x
13+
until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
14+
echo ""
15+
docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
16+
echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
17+
sleep 2;
18+
done;
19+
20+
# Always show logs if the container is running, this is very useful both on CI as well as while developing
21+
if container_running $1; then
22+
docker logs $1
23+
fi
24+
25+
if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
26+
cleanup_all_in_network $2
27+
echo
28+
echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
29+
echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
30+
return 1
31+
else
32+
echo
33+
echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
34+
return 0
35+
fi
36+
}

‎.buildkite/pipeline.yml

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,22 @@
11
steps:
2-
- label: ":python: Greetings"
3-
command: "echo 'Hello, world!'"
2+
- label: ":elasticsearch: :python: ES Python ({{ matrix.python }})"
3+
agents:
4+
provider: "gcp"
5+
env:
6+
PYTHON_VERSION: "{{ matrix.python }}"
7+
TEST_SUITE: "platinum"
8+
STACK_VERSION: 8.10.0-SNAPSHOT
9+
PYTHON_CONNECTION_CLASS: "{{ matrix.connection }}"
10+
matrix:
11+
setup:
12+
python:
13+
- "3.6"
14+
- "3.7"
15+
- "3.8"
16+
- "3.9"
17+
- "3.10"
18+
- "3.11"
19+
connection:
20+
- "urllib3"
21+
- "requests"
22+
command: ./.buildkite/run-tests

0 commit comments

Comments
 (0)