Buildkite test runner for REST test suite

With fixes for several test breakages.

Integration tests will always run against a real serverless instance
that has its own certs.

Author: JoshMock

.buildkite/rest-tests.yaml

@@ -0,0 +1,37 @@
+---
+steps:
+  - label: ":elasticsearch: :python: ES Serverless ({{ matrix.python }}/{{ matrix.connection_class }}) Python Test Suite: {{ matrix.suite }}"
+    agents:
+      provider: gcp
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      TEST_SUITE: "{{ matrix.suite }}"
+      PYTHON_CONNECTION_CLASS: "{{ matrix.connection_class }}"
+    matrix:
+      setup:
+        suite:
+          - free
+          - platinum
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        connection_class:
+          - urrlib3
+          - requests
+    command: ./.buildkite/run-tests
+    artifact_paths: "junit/*-junit.xml"
+  - wait: ~
+    continue_on_failure: true
+  - label: ":junit: Test results"
+    agents:
+      provider: gcp
+      image: family/core-ubuntu-2204
+    plugins:
+      - junit-annotate#v2.4.1:
+        artifacts: "junit/*-junit.xml"
+        job-uuid-file-pattern: "(.*)-junit.xml"
+        fail-build-on-error: true
+        failure-format: file

.buildkite/run-tests

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Default environment variables
+export TEST_SUITE="${TEST_SUITE:=platinum}"
+export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
+export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
+
+set -euo pipefail
+
+echo -e "--- :elasticsearch: Start serverless instance"
+# TODO
+
+echo -e "--- :computer: Environment variables"
+echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL"
+echo -e "STACK_VERSION $STACK_VERSION"
+echo -e "TEST_SUITE $TEST_SUITE"
+echo -e "PYTHON_VERSION $PYTHON_VERSION"
+echo -e "PYTHON_CONNECTION_CLASS $PYTHON_CONNECTION_CLASS"
+
+echo -e "--- :docker: Build elasticsearch-serverless-python container"
+
+docker build \
+  --file .ci/Dockerfile \
+  --tag elasticsearch-serverless-python \
+  --build-arg "PYTHON_VERSION=$PYTHON_VERSION" \
+  .
+
+echo -e "--- :docker: :python: Run integration tests for Python $PYTHON_VERSION"
+
+docker run \
+  --env STACK_VERSION \
+  --env ELASTICSEARCH_URL \
+  --env TEST_SUITE \
+  --env PYTHON_CONNECTION_CLASS \
+  --env ES_API_KEY \
+  --name elasticsearch-serverless-python-tests \
+  --volume "$(pwd)/junit:/code/elasticsearch-serverless-python/junit" \
+  --rm \
+  elasticsearch-serverless-python \
+  nox -s "test-$PYTHON_VERSION"
+
+echo -e "--- :elasticsearch: Tear down serverless instance"
+echo "TODO"

.ci/run-elasticsearch.sh

@@ -0,0 +1,42 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: elasticsearch-serverless-python
+spec:
+  type: library
+  owner: group:clients-team
+  lifecycle: production
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: elasticsearch-serverless-python-rest-tests
+  description: elasticsearch-serverless-python - rest tests
+spec:
+  type: buildkite-pipeline
+  owner: group:clients-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: elasticsearch-serverless-python - rest tests
+    spec:
+      repository: elastic/elasticsearch-serverless-python
+      pipeline_file: .buildkite/rest-tests.yaml
+      teams:
+        clients-team:
+          access_level: MANAGE_BUILD_AND_READ
+      provider_settings:
+        build_pull_requests: true
+        build_branches: true
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      schedules:
+        main_semi_daily:
+          branch: 'main'
+          cronline: '0 */12 * * *'

catalog-info.yaml

@@ -21,7 +21,7 @@
 
 from elasticsearch_serverless import Elasticsearch
 
-from .utils import CA_CERTS, es_url, es_version
+from .utils import es_url, es_version
 
 
 @pytest.fixture(scope="session")
@@ -33,11 +33,6 @@ def elasticsearch_url():
 
 
 @pytest.fixture(scope="session")
-def ca_certs():
-    return CA_CERTS
-
-
-@pytest.fixture(scope="session")
-def elasticsearch_version(elasticsearch_url, ca_certs) -> Tuple[int, ...]:
+def elasticsearch_version(elasticsearch_url) -> Tuple[int, ...]:
     """Returns the version of the current Elasticsearch cluster"""
-    return es_version(Elasticsearch(elasticsearch_url, ca_certs=ca_certs))
+    return es_version(Elasticsearch(elasticsearch_url))

test_elasticsearch_serverless/conftest.py

@@ -19,7 +19,7 @@
 
 import elasticsearch_serverless
 
-from ...utils import CA_CERTS, wipe_cluster
+from ...utils import wipe_cluster
 
 pytestmark = pytest.mark.asyncio
 
@@ -38,7 +38,7 @@ async def async_client(elasticsearch_url):
     client = None
     try:
         client = elasticsearch_serverless.AsyncElasticsearch(
-            elasticsearch_url, request_timeout=3, ca_certs=CA_CERTS
+            elasticsearch_url, request_timeout=3
         )
         yield client
     finally:

test_elasticsearch_serverless/test_async/test_server/conftest.py

@@ -21,7 +21,7 @@
 
 import elasticsearch_serverless
 
-from ..utils import CA_CERTS, wipe_cluster
+from ..utils import wipe_cluster, es_api_key
 
 # Information about the Elasticsearch instance running, if any
 # Used for
@@ -34,15 +34,15 @@
 def sync_client_factory(elasticsearch_url):
     client = None
     try:
-        # Configure the client with certificates and optionally
+        # Configure the client with API key and optionally
         # an HTTP conn class depending on 'PYTHON_CONNECTION_CLASS' envvar
-        kw = {"ca_certs": CA_CERTS}
+        kw = {}
         if "PYTHON_CONNECTION_CLASS" in os.environ:
             kw["node_class"] = os.environ["PYTHON_CONNECTION_CLASS"]
 
         # We do this little dance with the URL to force
         # Requests to respect 'headers: None' within rest API spec tests.
-        client = elasticsearch_serverless.Elasticsearch(elasticsearch_url, **kw)
+        client = elasticsearch_serverless.Elasticsearch(elasticsearch_url, api_key=es_api_key(), **kw)
 
         # Wipe the cluster before we start testing just in case it wasn't wiped
         # cleanly from the previous run of pytest?

test_elasticsearch_serverless/test_server/conftest.py

@@ -42,7 +42,7 @@
 from elasticsearch_serverless._sync.client.utils import _base64_auth_header
 from elasticsearch_serverless.compat import string_types
 
-from ..utils import CA_CERTS, es_url, parse_version
+from ..utils import es_url, es_api_key, parse_version
 
 # some params had to be changed in python, keep track of them so we can rename
 # those in the tests accordingly
@@ -555,7 +555,7 @@ def remove_implicit_resolver(cls, tag_to_remove):
 try:
     # Construct the HTTP and Elasticsearch client
     http = urllib3.PoolManager(retries=10)
-    client = Elasticsearch(es_url(), request_timeout=3, ca_certs=CA_CERTS)
+    client = Elasticsearch(es_url(), api_key=es_api_key(), request_timeout=3)
 
     # Make a request to Elasticsearch for the build hash, we'll be looking for
     # an artifact with this same hash to download test specs for.