diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile new file mode 100644 index 000000000..3c429a37b --- /dev/null +++ b/.buildkite/Dockerfile @@ -0,0 +1,20 @@ +ARG PYTHON_VERSION=3.8 +FROM python:${PYTHON_VERSION} + +# Default UID/GID to 1000 +# it can be overridden at build time +ARG BUILDER_UID=1000 +ARG BUILDER_GID=1000 +ENV BUILDER_USER elastic +ENV BUILDER_GROUP elastic +ENV PATH="${PATH}:/var/lib/elastic/.local/bin" + +# Create user +RUN groupadd --system -g ${BUILDER_GID} ${BUILDER_GROUP} \ + && useradd --system --shell /bin/bash -u ${BUILDER_UID} -g ${BUILDER_GROUP} -d /var/lib/elastic -m elastic 1>/dev/null 2>/dev/null \ + && mkdir -p /code/elasticsearch-py && mkdir /code/elasticsearch-py/build \ + && chown -R ${BUILDER_USER}:${BUILDER_GROUP} /code/ +WORKDIR /code/elasticsearch-py +USER ${BUILDER_USER}:${BUILDER_GROUP} +RUN python -m pip install --disable-pip-version-check nox +COPY --chown=$BUILDER_USER:$BUILDER_GROUP . . diff --git a/.buildkite/certs/ca.crt b/.buildkite/certs/ca.crt new file mode 100755 index 000000000..5ed1c9853 --- /dev/null +++ b/.buildkite/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/ +40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y +RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq +Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n +gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT +8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C +AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY +MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX +dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb +GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0 +hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj +cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx +EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI= +-----END CERTIFICATE----- diff --git a/.buildkite/certs/ca.pem b/.buildkite/certs/ca.pem new file mode 100644 index 000000000..5ed1c9853 --- /dev/null +++ b/.buildkite/certs/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/ +40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y +RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq +Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n +gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT +8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C +AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY +MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX +dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb +GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0 +hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj +cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx +EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI= +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode.crt b/.buildkite/certs/testnode.crt new file mode 100755 index 000000000..39eb092fa --- /dev/null +++ b/.buildkite/certs/testnode.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDODCCAiCgAwIBAgIVAKLWEcNzTd4B0NqnrJL0xAKaS8DWMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIzMDgyMTE3MjcwMloXDTI2MDgyMDE3MjcwMlowEzERMA8G +A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8 +eLXL3ZX5v8JlHcfg+96Bpq24EeiqV+7RPPKbcH80ODjkETqYUpam+TcOl2gt23p/ +rpiPSSpOX8pFdmY78wTmxo2GCQZ/db2h0gZOOYpb8HQku+hJ4bAmtzizrqWW76Wz +csen3DSUkT0bKkJTjUMmwVhRaMpfv8EIcUbrHAwc3VCj7grnFL0kdAuQa6iyBH4I +lTUYOIOVyEJ8zZ7R4BJO3QU+TRuJ5+w/QiZMeDqxtrdDL37vYQHPW7L/XISCCOMp +sA3avzFphoQXBQ8mjdB8Txkd4sH7mJTqnRp5ILhRzVpcPPgQYFeIB567B+kFeSau +aJJmc0EVgOcK5aSMtOH3AgMBAAGjYjBgMB0GA1UdDgQWBBQsZbZDudZ63h52FlU5 +N2g3pznkETAfBgNVHSMEGDAWgBR1YUax123eUBRIJN0RZ+GOC2KPTzATBgNVHREE +DDAKgghpbnN0YW5jZTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAyv0Cw +OrvZn7FHHS8TJI5vTi1F43R/eSNMNL/+q/nK93KaxWJH1T4zrJhrJ9KpzkFcalXP +bu02oTh28b3o3QpS2wdwMv/Q3NLoMBEmQlG2UrELFvV43nS8LCiwCX3o11L1HZP3 +1Z/rclwxbA4OQ/ZkPcol++TDZQTM/8WkIdZmTL4UDb/ppDjX24nTOitkMRZlYAOY +mid9GGExhKrUJ0I9/A3w1hWRA1Hwc+1TFDcPphl2x2uQ9HJFBueAvuFXmIjDki1x +qrvnFZ+mneI9kR4m82MX900WF15KS35GzmMui0tsf0wbfy3Jh+WnpMlIIa2OQXw7 +prbkg9tScQSsvhC8 +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode.key b/.buildkite/certs/testnode.key new file mode 100755 index 000000000..b7458996a --- /dev/null +++ b/.buildkite/certs/testnode.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAvHi1y92V+b/CZR3H4PvegaatuBHoqlfu0Tzym3B/NDg45BE6 +mFKWpvk3DpdoLdt6f66Yj0kqTl/KRXZmO/ME5saNhgkGf3W9odIGTjmKW/B0JLvo +SeGwJrc4s66llu+ls3LHp9w0lJE9GypCU41DJsFYUWjKX7/BCHFG6xwMHN1Qo+4K +5xS9JHQLkGuosgR+CJU1GDiDlchCfM2e0eASTt0FPk0biefsP0ImTHg6sba3Qy9+ +72EBz1uy/1yEggjjKbAN2r8xaYaEFwUPJo3QfE8ZHeLB+5iU6p0aeSC4Uc1aXDz4 +EGBXiAeeuwfpBXkmrmiSZnNBFYDnCuWkjLTh9wIDAQABAoIBAAU0iEDTI9s78pB8 +XBLYofKOuemFhRl/SDc7KbAlUT4N93RFDYs7bLG73Eto3xW1JBL2rXv3l1WGy71T +YctyEMaW4T28bhODGvOnK0lpyWp0n6CMGARCWW0YTlaYEjay866bEuyN5l3cDQX9 +Csvn8NzXJitJa51tXFVxW3YO1j7Nyc/M59oyBZ1ARYYmQqFYLEu6lvJOW0cKDFkZ +AcMVlOIxZQL/Mf+RO72aQGVuYNjqxlLIXLuE9zFR2gDFM2+l3FMUWDGHGBDFyjKU +iMk4+sSlOTFXqO9VQzua6FLFMsQT6m5PFD4uPY92KR6CPfH/NrWqwqr+jpjaU+gs +3U9GN+ECgYEA58qX7tKPk7CWdk3kyk5NsNcs/qib+heXWEubfhoU8LmSnbBQhOAz +wi//r/xm0OHGj84y66+G3T347iudrLjhr07oGM1QfjYT3kb90efLjwAfCECtyVYL +EQrWO5UeoTnmrhlB1mGL3sWaVAsVqNLz8i2H5c7sj0hxHsvM62159r8CgYEA0Cff +opJqmUpMpHm3sgjMWctylVrHBuQe5cl5Ad80pbd6mvtt4TvGXbUGNdzURfyve9DS +x1CVlj4Sz8VuelFQgYL+7/qUqZoms1aSgJpxWv8ou+wUHmlF3kVO8VKt3BNHV+8J +euSB6NG91BGguBoHgnOoVcjbDGdhJGRTojCNWskCgYEA1jE3nwDCnrbTA3XNk0ky +r9TXhmgm4r+EIpqTkL7nVOAXZVJ1xaQtECgsveKe3C2WwHLKSVMFbFMFQonZha+/ +FbHz9l9cH5U3XPL7QEpTp8xz4LtsHJ4/UbtS5vJQwKnxyjYaydGQYAb4KuunUz/F +H6kFaM6DeZB2v/+SWIfs6Z8CgYARUdAEyeP+vzTjVpFXSe4e5pOxI619wEtl2T6t +TjImO78C2DrLS9r0fxR2NNqgvCapybVQCj94EdAk44uOt+dX71thAluORRpFP8XO +14rpBGQSRtFhumaq2N95quR2dFAyW9xREmRQx+rgk1rpFplbXF48TQsU3CE0Evj2 +fM22KQKBgDhob7M9sWvXecxoyy3J17jUTcFqmqKcqGnx3ZJ7Q9CgAfjYqNNQws27 +wTuaJB0PEuCOu4t+lUHEBMIjGkBfo1bHd4EZaW04Xgbfn2j8MK2e+9GlRtedxxFg +c1JdRb5+eTgPwLcDsmMWIW357PDW7RDEI07G1ZB4SqxGTKkU7JOW +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/functions/cleanup.sh b/.buildkite/functions/cleanup.sh new file mode 100755 index 000000000..4c25166fb --- /dev/null +++ b/.buildkite/functions/cleanup.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Shared cleanup routines between different steps +# +# Please source .ci/functions/imports.sh as a whole not just this file +# +# Version 1.0.0 +# - Initial version after refactor + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + fi + if [[ -n "$1" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m" + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup_trap { + status=$? + set +x + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_all_in_network "$1" + fi + # status is 0 or SIGINT + if [[ "$status" == "0" || "$status" == "130" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +}; +function cleanup_all_in_network { + + if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then + echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m" + return 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network $1 + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" +}; diff --git a/.buildkite/functions/imports.sh b/.buildkite/functions/imports.sh new file mode 100755 index 000000000..e732ebba0 --- /dev/null +++ b/.buildkite/functions/imports.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Sets up all the common variables and imports relevant functions +# +# Version 1.0.1 +# - Initial version after refactor +# - Validate STACK_VERSION asap + +function require_stack_version() { + if [[ -z $STACK_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m" + exit 1 + fi +} + +require_stack_version + +if [[ -z $es_node_name ]]; then + # only set these once + set -euo pipefail + export TEST_SUITE=${TEST_SUITE-platinum} + export RUNSCRIPTS=${RUNSCRIPTS-} + export DETACH=${DETACH-false} + export CLEANUP=${CLEANUP-false} + + export es_node_name=instance + export elastic_password=changeme + export elasticsearch_image=elasticsearch + export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200 + if [[ $TEST_SUITE != "platinum" ]]; then + export elasticsearch_url=http://${es_node_name}:9200 + fi + export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost} + export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}" + + export suffix=rest-test + export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-') + export network_name=${moniker}${suffix} + + export ssl_cert="${script_path}/certs/testnode.crt" + export ssl_key="${script_path}/certs/testnode.key" + export ssl_ca="${script_path}/certs/ca.crt" + +fi + + export script_path=$(dirname $(realpath -s $0)) + source $script_path/functions/cleanup.sh + source $script_path/functions/wait-for-container.sh + trap "cleanup_trap ${network_name}" EXIT + + +if [[ "$CLEANUP" == "true" ]]; then + cleanup_all_in_network $network_name + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m" +docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name" + diff --git a/.buildkite/functions/wait-for-container.sh b/.buildkite/functions/wait-for-container.sh new file mode 100755 index 000000000..1a721b588 --- /dev/null +++ b/.buildkite/functions/wait-for-container.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Exposes a routine scripts can call to wait for a container if that container set up a health command +# +# Please source .ci/functions/imports.sh as a whole not just this file +# +# Version 1.0.1 +# - Initial version after refactor +# - Make sure wait_for_contiainer is silent + +function wait_for_container { + set +x + until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1} + echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m" + sleep 2; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running $1; then + docker logs $1 + fi + + if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then + cleanup_all_in_network $2 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + return 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m" + return 0 + fi +} diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ffb9ebab5..fe1f9a1be 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,21 @@ steps: - - label: ":python: Greetings" - command: "echo 'Hello, world!'" + - label: ":elasticsearch: :python: ES Python ({{ matrix.python }})" + agents: + provider: "gcp" + env: + PYTHON_VERSION: "{{ matrix.python }}" + TEST_SUITE: "platinum" + STACK_VERSION: 8.10.0-SNAPSHOT + PYTHON_CONNECTION_CLASS: "{{ matrix.connection }}" + matrix: + setup: + python: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + connection: + - "urllib3" + - "requests" + command: ./.buildkite/run-tests diff --git a/.buildkite/run-elasticsearch.sh b/.buildkite/run-elasticsearch.sh new file mode 100755 index 000000000..2f73ea8d1 --- /dev/null +++ b/.buildkite/run-elasticsearch.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'. +# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'. +# Export the NUMBER_OF_NODES variable to start more than 1 node + +# Version 1.6.0 +# - Initial version of the run-elasticsearch.sh script +# - Deleting the volume should not dependent on the container still running +# - Fixed `ES_JAVA_OPTS` config +# - Moved to STACK_VERSION and TEST_VERSION +# - Refactored into functions and imports +# - Support NUMBER_OF_NODES +# - Added 5 retries on docker pull for fixing transient network errors +# - Added flags to make local CCR configurations work +# - Added action.destructive_requires_name=false as the default will be true in v8 +# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing +# - Moved ELASTIC_PASSWORD and xpack.security.enabled to the base arguments for "Security On by default" +# - Use https only when TEST_SUITE is "platinum", when "free" use http + +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m" +cleanup_node $es_node_name + +master_node_name=${es_node_name} +cluster_name=${moniker}${suffix} + +BUILDKITE=${BUILDKITE-false} +# Set vm.max_map_count kernel setting to 262144 if we're in CI +if [[ "$BUILDKITE" == "true" ]]; then + sudo sysctl -w vm.max_map_count=262144 +fi + +declare -a volumes +environment=($(cat <<-END + --env ELASTIC_PASSWORD=$elastic_password + --env xpack.security.enabled=true + --env node.name=$es_node_name + --env cluster.name=$cluster_name + --env cluster.initial_master_nodes=$master_node_name + --env discovery.seed_hosts=$master_node_name + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* + --env action.destructive_requires_name=false + --env ingest.geoip.downloader.enabled=false + --env cluster.deprecation_indexing.enabled=false +END +)) +if [[ "$TEST_SUITE" == "platinum" ]]; then + environment+=($(cat <<-END + --env xpack.license.self_generated.type=trial + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.verification_mode=certificate + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key + --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt +END +)) +else + environment+=($(cat <<-END + --env xpack.security.http.ssl.enabled=false +END +)) +fi + +cert_validation_flags="" +if [[ "$TEST_SUITE" == "platinum" ]]; then + cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1" +fi + +# Pull the container, retry on failures up to 5 times with +# short delays between each attempt. Fixes most transient network errors. +docker_pull_attempts=0 +until [ "$docker_pull_attempts" -ge 5 ] +do + docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break + docker_pull_attempts=$((docker_pull_attempts+1)) + echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..." + sleep 10 +done + +NUMBER_OF_NODES=${NUMBER_OF_NODES-1} +http_port=9200 +for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do + node_name=${es_node_name}$i + node_url=${external_elasticsearch_url/9200/${http_port}}$i + if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi + environment+=($(cat <<-END + --env node.name=$node_name +END +)) + echo "$i: $http_port $node_url " + volume_name=${node_name}-${suffix}-data + volumes+=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data${i} +END +)) + + # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started + local_detach="true" + if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi + echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m" + set -x + docker run \ + -u "$(id -u)" \ + --name "$node_name" \ + --network "$network_name" \ + --env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$http_port":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$local_detach" \ + --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$elasticsearch_container"; + + set +x + if wait_for_container "$es_node_name" "$network_name"; then + echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m" + fi + +done + diff --git a/.buildkite/run-nox.sh b/.buildkite/run-nox.sh new file mode 100755 index 000000000..ab8f4be01 --- /dev/null +++ b/.buildkite/run-nox.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [[ -z "$NOX_SESSION" ]]; then + NOX_SESSION=test-${PYTHON_VERSION%-dev} +fi +nox -s $NOX_SESSION diff --git a/.buildkite/run-repository.sh b/.buildkite/run-repository.sh new file mode 100755 index 000000000..466b906cc --- /dev/null +++ b/.buildkite/run-repository.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Called by entry point `run-test` use this script to add your repository specific test commands +# Once called Elasticsearch is up and running and the following parameters are available to this script + +# ELASTICSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease) +# ELASTICSEARCH_CONTAINER -- the docker moniker as a reference to know which docker image distribution is used +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable +# NETWORK_NAME -- The docker network name +# NODE_NAME -- The docker container name also used as Elasticsearch node name + +# When run in CI the test-matrix is used to define additional variables +# TEST_SUITE -- either `oss` or `xpack`, defaults to `oss` in `run-tests` + +set -e + +echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m" +echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLASS}\033[0m" + +echo -e "\033[1m>>>>> Build [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build \ + --file .buildkite/Dockerfile \ + --tag elastic/elasticsearch-py \ + --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \ + --build-arg "BUILDER_UID=$(id -u)" \ + --build-arg "BUILDER_GID=$(id -g)" \ + . + +echo -e "\033[1m>>>>> Run [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +mkdir -p junit +docker run \ + -u "$(id -u):$(id -g)" \ + --network=${network_name} \ + --env "STACK_VERSION=${STACK_VERSION}" \ + --env "ELASTICSEARCH_URL=${elasticsearch_url}" \ + --env "TEST_SUITE=${TEST_SUITE}" \ + --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \ + --env "TEST_TYPE=server" \ + --name elasticsearch-py \ + --rm \ + elastic/elasticsearch-py \ + nox -s test-${PYTHON_VERSION} diff --git a/.buildkite/run-tests b/.buildkite/run-tests new file mode 100755 index 000000000..7e7339fb6 --- /dev/null +++ b/.buildkite/run-tests @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Version 1.1 +# - Moved to .ci folder and seperated out `run-repository.sh` +# - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products + +# Default environment variables +export STACK_VERSION="${STACK_VERSION:=8.0.0-SNAPSHOT}" +export TEST_SUITE="${TEST_SUITE:=platinum}" +export PYTHON_VERSION="${PYTHON_VERSION:=3.9}" +export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}" + +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo "--- :elasticsearch: Starting Elasticsearch" +DETACH=true bash $script_path/run-elasticsearch.sh + +if [[ -n "$RUNSCRIPTS" ]]; then + for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do + echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + CONTAINER_NAME=${RUNSCRIPT} \ + DETACH=true \ + bash $script_path/run-${RUNSCRIPT}.sh + done +fi + +echo "+++ :python: Client tests" +bash $script_path/run-repository.sh