Run OpenTelemetry integration tests separately

Author: pquentin

.buildkite/generatesteps.py

@@ -3,7 +3,7 @@
 import yaml
 
 
-def benchmark_to_steps(python, connection_class):
+def benchmark_to_steps(python, connection_class, nox_session):
     return [
         {
             "group": f":elasticsearch: :python: ES Serverless ({python}/{connection_class})",
@@ -14,6 +14,7 @@ def benchmark_to_steps(python, connection_class):
                     "env": {
                         "PYTHON_VERSION": f"{python}",
                         "PYTHON_CONNECTION_CLASS": f"{connection_class}",
+                        "NOX_SESSION": f"{nox_session}",
                         # For development versions
                         # https://github.com/aio-libs/aiohttp/issues/6600
                         "AIOHTTP_NO_EXTENSIONS": 1,
@@ -55,5 +56,7 @@ def benchmark_to_steps(python, connection_class):
     steps = []
     for python in ["3.9", "3.10", "3.11", "3.12"]:
         for connection_class in ["urllib3", "requests"]:
-            steps.extend(benchmark_to_steps(python, connection_class))
+            steps.extend(benchmark_to_steps(python, connection_class, "test"))
+    steps.extend(benchmark_to_steps("3.9", "urllib3", "test_otel"))
+    steps.extend(benchmark_to_steps("3.12", "urllib3", "test_otel"))
     print(yaml.dump({"steps": steps}, Dumper=yaml.Dumper, sort_keys=False))

.buildkite/run-tests

@@ -51,6 +51,7 @@ echo -e "--- :computer: Environment variables"
 echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL"
 echo -e "PYTHON_VERSION $PYTHON_VERSION"
 echo -e "PYTHON_CONNECTION_CLASS $PYTHON_CONNECTION_CLASS"
+echo -e "NOX_SESSION $NOX_SESSION"
 
 echo -e "--- :docker: Build elasticsearch-serverless-python container"
 
@@ -74,4 +75,4 @@ docker run \
   --volume "$(pwd)/junit:/code/elasticsearch-serverless-python/junit" \
   --rm \
   elasticsearch-serverless-python \
-  nox -s "test-$PYTHON_VERSION"
+  nox -s ${NOX_SESSION}-${PYTHON_VERSION}

noxfile.py

@@ -30,15 +30,11 @@
 # Allow building aiohttp when no wheels are available (eg. for recent Python versions)
 INSTALL_ENV = {"AIOHTTP_NO_EXTENSIONS": "1"}
 
-
-@nox.session(python=["3.9", "3.10", "3.11", "3.12"])
-def test(session):
-    session.install(".[dev]", env=INSTALL_ENV)
-
+def pytest_argv():
     junit_xml = os.path.join(
         SOURCE_DIR, "junit", "elasticsearch-serverless-python-junit.xml"
     )
-    pytest_argv = [
+    return [
         "pytest",
         "--cov-report=term-missing",
         "--cov=elasticsearch_serverless",
@@ -47,10 +43,21 @@ def test(session):
         "--log-level=debug",
         "-vv",
         "--cache-clear",
-        *(session.posargs),
     ]
-    session.run(*pytest_argv)
 
+@nox.session(python=["3.9", "3.10", "3.11", "3.12"])
+def test(session):
+    session.install(".[dev]", env=INSTALL_ENV)
+
+    session.run(*pytest_argv(), *(session.posargs))
+
+@nox.session(python=["3.9", "3.12"])
+def test_otel(session):
+    session.install(".[dev]", env=INSTALL_ENV)
+    session.install("opentelemetry-api", "opentelemetry-sdk")
+
+    argv = pytest_argv() + ["-m", "otel"]
+    session.run(*argv, *(session.posargs), env={"TEST_WITH_OTEL": "1"})
 
 @nox.session()
 def format(session):

pyproject.toml

@@ -69,8 +69,6 @@ dev = [
     "numpy",
     "pandas",
     "mapbox-vector-tile",
-    "opentelemetry-api",
-    "opentelemetry-sdk",
 ]
 docs = [
     "sphinx-rtd-theme>=1.2.2",
@@ -101,7 +99,8 @@ packages = ["elasticsearch_serverless"]
 
 [tool.pytest.ini_options]
 junit_family = "legacy"
-xfail_strict=true
+xfail_strict = true
+markers = "otel"
 
 [tool.isort]
 profile = "black"

test_elasticsearch_serverless/test_server/test_otel.py

@@ -15,37 +15,49 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider, export
-from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+import os
 
-import elasticsearch
-from test_elasticsearch.utils import CA_CERTS
+import pytest
 
+import elasticsearch_serverless
 
-def test_otel_end_to_end(monkeypatch, elasticsearch_url: str):
+try:
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider, export
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+        InMemorySpanExporter,
+    )
+except ModuleNotFoundError:
+    pass
+
+pytestmark = [
+    pytest.mark.skipif(
+        "TEST_WITH_OTEL" not in os.environ, reason="TEST_WITH_OTEL is not set"
+    ),
+    pytest.mark.otel,
+]
+
+
+def test_otel_end_to_end(monkeypatch, sync_client):
     # Sets the global default tracer provider
     tracer_provider = TracerProvider()
     memory_exporter = InMemorySpanExporter()
     span_processor = export.SimpleSpanProcessor(memory_exporter)
     tracer_provider.add_span_processor(span_processor)
     trace.set_tracer_provider(tracer_provider)
 
-    # Once OpenTelemetry is enabled by default, we can use the sync_client fixture instead
-    monkeypatch.setenv("OTEL_PYTHON_INSTRUMENTATION_ELASTICSEARCH_ENABLED", "true")
-    kw = {}
-    if elasticsearch_url.startswith("https://"):
-        kw["ca_certs"] = CA_CERTS
-    client = elasticsearch.Elasticsearch(elasticsearch_url, **kw)
-
-    resp = client.search(index="logs-*", query={"match_all": {}})
+    resp = sync_client.search(index="logs-*", query={"match_all": {}})
     assert resp.meta.status == 200
 
     spans = memory_exporter.get_finished_spans()
     assert len(spans) == 1
     assert spans[0].name == "search"
-    assert spans[0].attributes == {
+    expected_attributes = {
         "http.request.method": "POST",
         "db.system": "elasticsearch",
+        "db.operation": "search",
         "db.elasticsearch.path_parts.index": "logs-*",
     }
+    # Assert expected atttributes are here, but allow other attributes too
+    # to make this test robust to elastic-transport changes
+    assert expected_attributes.items() <= spans[0].attributes.items()