From 296fc95baea76acf8f944c7592c3926e9de0dd34 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 24 Aug 2023 15:42:25 +0400 Subject: [PATCH 1/4] Inline elasticsearch.helpers.tests in conftest.py The test helpers were removed in elasticsearch-py 8 anyway, and this will help re-enabling integration tests in CI. --- tests/conftest.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 09c092890..d85070460 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,11 +18,14 @@ import os import re +import time from datetime import datetime +from unittest import SkipTest, TestCase from unittest.mock import Mock +from elasticsearch import Elasticsearch +from elasticsearch.exceptions import ConnectionError from elasticsearch.helpers import bulk -from elasticsearch.helpers.test import SkipTest, get_test_client from pytest import fixture, skip from elasticsearch_dsl.connections import add_connection, connections @@ -36,6 +39,73 @@ ) from .test_integration.test_document import Comment, History, PullRequest, User +if "ELASTICSEARCH_URL" in os.environ: + ELASTICSEARCH_URL = os.environ["ELASTICSEARCH_URL"] +else: + ELASTICSEARCH_URL = "https://elastic:changeme@localhost:9200" + + +def get_test_client(nowait=False, **kwargs): + # construct kwargs from the environment + kw = {"timeout": 30} + + if "PYTHON_CONNECTION_CLASS" in os.environ: + from elasticsearch import connection + + kw["connection_class"] = getattr( + connection, os.environ["PYTHON_CONNECTION_CLASS"] + ) + + kw.update(kwargs) + client = Elasticsearch(ELASTICSEARCH_URL, **kw) + + # wait for yellow status + for _ in range(1 if nowait else 100): + try: + client.cluster.health(wait_for_status="yellow") + return client + except ConnectionError: + time.sleep(0.1) + else: + # timeout + raise SkipTest("Elasticsearch failed to start.") + + +class ElasticsearchTestCase(TestCase): + @staticmethod + def _get_client(): + return get_test_client() + + @classmethod + def setup_class(cls): + cls.client = cls._get_client() + + def teardown_method(self, _): + # Hidden indices expanded in wildcards in ES 7.7 + expand_wildcards = ["open", "closed"] + if self.es_version() >= (7, 7): + expand_wildcards.append("hidden") + + self.client.indices.delete_data_stream( + name="*", ignore=404, expand_wildcards=expand_wildcards + ) + self.client.indices.delete( + index="*", ignore=404, expand_wildcards=expand_wildcards + ) + self.client.indices.delete_template(name="*", ignore=404) + + def es_version(self): + if not hasattr(self, "_es_version"): + self._es_version = _get_version(client.info()["version"]["number"]) + return self._es_version + + +def _get_version(version_string): + if "." not in version_string: + return () + version = version_string.strip().split(".") + return tuple(int(v) if v.isdigit() else 999 for v in version) + @fixture(scope="session") def client(): From 90a5e61d2fefae9c8d9b4dd6bf7d10f312e4ae92 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 24 Aug 2023 15:54:06 +0400 Subject: [PATCH 2/4] Rename nowait to wait for clarity --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d85070460..295f6db61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ ELASTICSEARCH_URL = "https://elastic:changeme@localhost:9200" -def get_test_client(nowait=False, **kwargs): +def get_test_client(wait=True, **kwargs): # construct kwargs from the environment kw = {"timeout": 30} @@ -60,7 +60,7 @@ def get_test_client(nowait=False, **kwargs): client = Elasticsearch(ELASTICSEARCH_URL, **kw) # wait for yellow status - for _ in range(1 if nowait else 100): + for _ in range(100 if wait else 1): try: client.cluster.health(wait_for_status="yellow") return client @@ -110,7 +110,7 @@ def _get_version(version_string): @fixture(scope="session") def client(): try: - connection = get_test_client(nowait="WAIT_FOR_ES" not in os.environ) + connection = get_test_client(wait="WAIT_FOR_ES" in os.environ) add_connection("default", connection) return connection except SkipTest: From 14baddfe31180cc97b1fdfb052158866846b92c7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 24 Aug 2023 16:00:55 +0400 Subject: [PATCH 3/4] Fail loudly if we can't connect and WAIT_FOR_ES=1 --- tests/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 295f6db61..c1e6d7106 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,15 +60,16 @@ def get_test_client(wait=True, **kwargs): client = Elasticsearch(ELASTICSEARCH_URL, **kw) # wait for yellow status - for _ in range(100 if wait else 1): + for tries_left in range(100 if wait else 1, 0, -1): try: client.cluster.health(wait_for_status="yellow") return client except ConnectionError: + if wait and tries_left == 1: + raise time.sleep(0.1) - else: - # timeout - raise SkipTest("Elasticsearch failed to start.") + + raise SkipTest("Elasticsearch failed to start.") class ElasticsearchTestCase(TestCase): From c964f1f58884e608398d01a74623140f0c113a77 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 24 Aug 2023 11:49:43 +0400 Subject: [PATCH 4/4] Run integration tests in CI --- .github/workflows/ci.yml | 2 ++ tests/conftest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eade8354..5cd05d38f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,5 @@ jobs: - name: Run Tests run: | nox -rs test-${{ matrix.python-version }} + env: + WAIT_FOR_ES: "1" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index c1e6d7106..8400ed013 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ if "ELASTICSEARCH_URL" in os.environ: ELASTICSEARCH_URL = os.environ["ELASTICSEARCH_URL"] else: - ELASTICSEARCH_URL = "https://elastic:changeme@localhost:9200" + ELASTICSEARCH_URL = "http://localhost:9200" def get_test_client(wait=True, **kwargs):