From 9e48dc27c4e3c4f46b35801fbf1cbaf3017fceb8 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Wed, 23 Aug 2023 16:30:03 -0600 Subject: [PATCH 01/14] update Jenkins matrix to use available python versions in the latest runner image --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e6c2d9700f..bd61d4babb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ slack = new Slack() DEFAULT_CASSANDRA = ['2.1', '2.2', '3.0', '3.11', '4.0'] DEFAULT_DSE = ['dse-5.0.15', 'dse-5.1.35', 'dse-6.0.18', 'dse-6.7.17', 'dse-6.8.30'] -DEFAULT_RUNTIME = ['3.7.7', '3.8.3'] +DEFAULT_RUNTIME = ['3.8.16', '3.9.16', '3.10.11', '3.11.3'] DEFAULT_CYTHON = ["True", "False"] matrices = [ "FULL": [ @@ -620,8 +620,8 @@ pipeline { triggers { parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ # Every weeknight (Monday - Friday) around 4:00 AM - # These schedules will run with and without Cython enabled for Python 3.7.7 and 3.8.3 - H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.7.7 3.8.3;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1.35 dse-6.0.18 dse-6.7.17 + # These schedules will run with and without Cython enabled for Python 3.8.16 and 3.11.3 + H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.11.3;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1.35 dse-6.0.18 dse-6.7.17 """ : "") } From ae4b5bd276aca7737e1d1a4dc7c046adafb5430b Mon Sep 17 00:00:00 2001 From: weideng1 Date: Wed, 23 Aug 2023 20:31:15 -0600 Subject: [PATCH 02/14] Use earliest and latest supported python runtimes for smoke test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd61d4babb..662a564e79 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,7 +52,7 @@ matrices = [ ], "SMOKE": [ "SERVER": DEFAULT_CASSANDRA.takeRight(2) + DEFAULT_DSE.takeRight(1), - "RUNTIME": DEFAULT_RUNTIME.takeRight(2), + "RUNTIME": DEFAULT_RUNTIME.take(1) + DEFAULT_RUNTIME.takeRight(1), "CYTHON": ["True"] ] ] From c8bebf696e717e3451fb66efc9cc07634d650bcd Mon Sep 17 00:00:00 2001 From: weideng1 Date: Wed, 23 Aug 2023 21:59:51 -0600 Subject: [PATCH 03/14] Remove C* and DSE versions that are EOL'ed --- Jenkinsfile | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 662a564e79..7eec06b043 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,8 +30,8 @@ import com.datastax.jenkins.drivers.python.Slack slack = new Slack() -DEFAULT_CASSANDRA = ['2.1', '2.2', '3.0', '3.11', '4.0'] -DEFAULT_DSE = ['dse-5.0.15', 'dse-5.1.35', 'dse-6.0.18', 'dse-6.7.17', 'dse-6.8.30'] +DEFAULT_CASSANDRA = ['3.0', '3.11', '4.0'] +DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30'] DEFAULT_RUNTIME = ['3.8.16', '3.9.16', '3.10.11', '3.11.3'] DEFAULT_CYTHON = ["True", "False"] matrices = [ @@ -502,14 +502,6 @@ pipeline { DEFAULT Default to the build context. - - 2.1 - Apache CassandraⓇ; v2.1.x - - - 2.2 - Apache CassandarⓇ; v2.2.x - 3.0 Apache CassandraⓇ v3.0.x @@ -522,22 +514,10 @@ pipeline { 4.0 Apache CassandraⓇ v4.0.x - - dse-5.0.15 - DataStax Enterprise v5.0.x (Long Term Support) - dse-5.1.35 DataStax Enterprise v5.1.x - - dse-6.0.18 - DataStax Enterprise v6.0.x - - - dse-6.7.17 - DataStax Enterprise v6.7.x - dse-6.8.30 DataStax Enterprise v6.8.x (CURRENTLY UNDER DEVELOPMENT) @@ -621,7 +601,7 @@ pipeline { parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ # Every weeknight (Monday - Friday) around 4:00 AM # These schedules will run with and without Cython enabled for Python 3.8.16 and 3.11.3 - H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.11.3;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1.35 dse-6.0.18 dse-6.7.17 + H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.11.3;CI_SCHEDULE_SERVER_VERSION=3.11 4.0 dse-5.1.35 dse-6.8.30 """ : "") } From 960336f1d7f0f04fa8315ffc8d21f5ec97767eaf Mon Sep 17 00:00:00 2001 From: weideng1 Date: Wed, 23 Aug 2023 22:30:45 -0600 Subject: [PATCH 04/14] updated ccm and PyYAML version to avoid problem with higher python versions --- test-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 996cf4341f..349a7a2250 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,8 @@ scales nose mock>1.1 -ccm>=2.1.2 +ccm>=2.1.5 +PyYAML>=6.0.1 pytz sure pure-sasl From 79933591800e73b9dcc437b2b685a70ac37dc9ac Mon Sep 17 00:00:00 2001 From: weideng1 Date: Wed, 23 Aug 2023 23:48:22 -0600 Subject: [PATCH 05/14] workaround for CCM's PyYAML dependency that's not compatible with higher python versions --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 7eec06b043..0aa03af310 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -167,6 +167,7 @@ def initializeEnvironment() { sudo apt-get install socat pip install --upgrade pip pip install -U setuptools + sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt pip install ${HOME}/ccm ''' From f79ed893ccf83d306f6af912152f9a1a0ec4b978 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Thu, 24 Aug 2023 00:00:47 -0600 Subject: [PATCH 06/14] fix setup.py that also has pyYaml version requirement, causing python 3.10+ problem --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0aa03af310..698fd21877 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,6 +168,7 @@ def initializeEnvironment() { pip install --upgrade pip pip install -U setuptools sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt + sed -i -e "s/\(pyYaml\) < [0-9]*\.[0-9]*'/\1'/" setup.py pip install ${HOME}/ccm ''' From cb90f6eee8bb319b4d421ef18443dc39dbb005ef Mon Sep 17 00:00:00 2001 From: weideng1 Date: Thu, 24 Aug 2023 00:04:02 -0600 Subject: [PATCH 07/14] escape backslash to avoid groovy freaking out --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 698fd21877..9be681ae80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,7 +168,7 @@ def initializeEnvironment() { pip install --upgrade pip pip install -U setuptools sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt - sed -i -e "s/\(pyYaml\) < [0-9]*\.[0-9]*'/\1'/" setup.py + sed -i -e "s/\\(pyYaml\\) < [0-9]*\\.[0-9]*'/\\1'/" setup.py pip install ${HOME}/ccm ''' From b00f3b864ab842040f58b719771a55763c677480 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Thu, 24 Aug 2023 00:10:59 -0600 Subject: [PATCH 08/14] simplify sed script to fix setup.py --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9be681ae80..ded58a3a0f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,7 +168,7 @@ def initializeEnvironment() { pip install --upgrade pip pip install -U setuptools sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt - sed -i -e "s/\\(pyYaml\\) < [0-9]*\\.[0-9]*'/\\1'/" setup.py + sed -i -e "s/pyYaml < 6.0/pyYaml/" setup.py pip install ${HOME}/ccm ''' From f851b4642d33b87b7c3551404ad8cd056c34b1f7 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Thu, 24 Aug 2023 00:15:41 -0600 Subject: [PATCH 09/14] fix typo in the sed script to fix setup.py --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ded58a3a0f..068a58302c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,7 +168,7 @@ def initializeEnvironment() { pip install --upgrade pip pip install -U setuptools sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt - sed -i -e "s/pyYaml < 6.0/pyYaml/" setup.py + sed -i -e "s/pyYaml < 6.0/pyYaml/" ${HOME}/ccm/setup.py pip install ${HOME}/ccm ''' From b1a7a0168cdc9cd7d3ce6928accdb0a015c540f2 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Fri, 25 Aug 2023 12:32:52 -0600 Subject: [PATCH 10/14] address Bret's comments --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 068a58302c..999b967687 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -167,9 +167,8 @@ def initializeEnvironment() { sudo apt-get install socat pip install --upgrade pip pip install -U setuptools - sed -i -e 's/pyyaml.*/pyyaml/' ${HOME}/ccm/requirements.txt - sed -i -e "s/pyYaml < 6.0/pyYaml/" ${HOME}/ccm/setup.py pip install ${HOME}/ccm + pip install "pyyaml > 6.0" ''' // Determine if server version is Apache CassandraⓇ or DataStax Enterprise From 0ca0ddf874eb0765be53d09b18379b6427571d25 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Fri, 25 Aug 2023 16:07:27 -0600 Subject: [PATCH 11/14] second attempt to make ccm-private work with python 3.10+ --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 999b967687..9e7a70d2c8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -167,8 +167,9 @@ def initializeEnvironment() { sudo apt-get install socat pip install --upgrade pip pip install -U setuptools + pip install wheel + pip install "Cython<3.0" "pyyaml<6.0" --no-build-isolation pip install ${HOME}/ccm - pip install "pyyaml > 6.0" ''' // Determine if server version is Apache CassandraⓇ or DataStax Enterprise From 59b36cefee63d1815a82001bbd2aaa2862104971 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Mon, 28 Aug 2023 08:31:15 -0600 Subject: [PATCH 12/14] use pynose to replace nosetests so that it can run under Python 3.10+, before PYTHON-1297 gets merged --- Jenkinsfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9e7a70d2c8..46654ffff7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -188,7 +188,7 @@ def initializeEnvironment() { } sh label: 'Install unit test modules', script: '''#!/bin/bash -lex - pip install nose-ignore-docstring nose-exclude service_identity + pip install pynose nose-ignore-docstring nose-exclude service_identity ''' if (env.CYTHON_ENABLED == 'True') { @@ -249,9 +249,9 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_results.xml tests/unit/ || true - EVENT_LOOP=eventlet VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_eventlet_results.xml tests/unit/io/test_eventletreactor.py || true - EVENT_LOOP=gevent VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_gevent_results.xml tests/unit/io/test_geventreactor.py || true + EVENT_LOOP=${EVENT_LOOP} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_results.xml tests/unit/ || true + EVENT_LOOP=eventlet VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_eventlet_results.xml tests/unit/io/test_eventletreactor.py || true + EVENT_LOOP=gevent VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_gevent_results.xml tests/unit/io/test_geventreactor.py || true ''' sh label: 'Execute Simulacron integration tests', script: '''#!/bin/bash -lex @@ -261,13 +261,13 @@ def executeStandardTests() { set +o allexport SIMULACRON_JAR="${HOME}/simulacron.jar" - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_results.xml tests/integration/simulacron/ || true + SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_results.xml tests/integration/simulacron/ || true # Run backpressure tests separately to avoid memory issue - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_1_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_paused_connections || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_2_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_queued_requests_timeout || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_3_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_cluster_busy || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_4_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_node_busy || true + SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_1_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_paused_connections || true + SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_2_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_queued_requests_timeout || true + SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_3_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_cluster_busy || true + SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_4_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_node_busy || true ''' sh label: 'Execute CQL engine integration tests', script: '''#!/bin/bash -lex @@ -276,7 +276,7 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=cqle_results.xml tests/integration/cqlengine/ || true + EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=cqle_results.xml tests/integration/cqlengine/ || true ''' sh label: 'Execute Apache CassandraⓇ integration tests', script: '''#!/bin/bash -lex @@ -285,7 +285,7 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true + EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true ''' if (env.CASSANDRA_VERSION.split('-')[0] == 'dse' && env.CASSANDRA_VERSION.split('-')[1] != '4.8') { @@ -295,7 +295,7 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} DSE_VERSION=${DSE_VERSION} ADS_HOME="${HOME}/" VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true + EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} DSE_VERSION=${DSE_VERSION} ADS_HOME="${HOME}/" VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true ''' } @@ -305,7 +305,7 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CLOUD_PROXY_PATH="${HOME}/proxy/" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=advanced_results.xml tests/integration/cloud/ || true + EVENT_LOOP=${EVENT_LOOP} CLOUD_PROXY_PATH="${HOME}/proxy/" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=advanced_results.xml tests/integration/cloud/ || true ''' if (env.PROFILE == 'FULL') { @@ -315,7 +315,7 @@ def executeStandardTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --exclude-dir=tests/integration/long/upgrade --with-ignore-docstrings --with-xunit --xunit-file=long_results.xml tests/integration/long/ || true + EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --exclude-dir=tests/integration/long/upgrade --with-ignore-docstrings --with-xunit --xunit-file=long_results.xml tests/integration/long/ || true ''' } } @@ -327,7 +327,7 @@ def executeDseSmokeTests() { . ${HOME}/environment.txt set +o allexport - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} DSE_VERSION=${DSE_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/test_dse.py || true + EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} DSE_VERSION=${DSE_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/test_dse.py || true ''' } @@ -348,7 +348,7 @@ def executeEventLoopTests() { "tests/integration/simulacron/test_endpoint.py" "tests/integration/long/test_ssl.py" ) - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml ${EVENT_LOOP_TESTS[@]} || true + EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} pynose -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml ${EVENT_LOOP_TESTS[@]} || true ''' } From a3051f332a850daf7864e4089dfa6b5118e00c4e Mon Sep 17 00:00:00 2001 From: weideng1 Date: Mon, 28 Aug 2023 13:04:13 -0600 Subject: [PATCH 13/14] address issue that brought in the wrong ccm-public --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 349a7a2250..996cf4341f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,8 +2,7 @@ scales nose mock>1.1 -ccm>=2.1.5 -PyYAML>=6.0.1 +ccm>=2.1.2 pytz sure pure-sasl From 6c97c735585f45efe556f5c2a99e4da7fe3e6529 Mon Sep 17 00:00:00 2001 From: weideng1 Date: Mon, 28 Aug 2023 14:21:29 -0600 Subject: [PATCH 14/14] added a comment as Bret suggested --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 46654ffff7..a6d19fb5ef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -167,6 +167,9 @@ def initializeEnvironment() { sudo apt-get install socat pip install --upgrade pip pip install -U setuptools + + # install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023 + # this works around the python-3.10+ compatibility problem as described in DSP-23524 pip install wheel pip install "Cython<3.0" "pyyaml<6.0" --no-build-isolation pip install ${HOME}/ccm