From 29006e25d3338e2cc92edb5406cc4e3b665ad630 Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Wed, 23 Nov 2022 12:37:38 +0100 Subject: [PATCH 1/4] (Temporarily) disable failing Mapper test Disable should_allow_less_parameters_than_bind_markers_if_there_are_repeated_names test due to scylladb/scylladb#11945 bug. This is temporary: either Scylla should get fixed or driver should work around the new implementation. --- .../com/datastax/driver/mapping/MapperAccessorParamsTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver-mapping/src/test/java/com/datastax/driver/mapping/MapperAccessorParamsTest.java b/driver-mapping/src/test/java/com/datastax/driver/mapping/MapperAccessorParamsTest.java index 7e29dd49c80..5f719abe4b6 100644 --- a/driver-mapping/src/test/java/com/datastax/driver/mapping/MapperAccessorParamsTest.java +++ b/driver-mapping/src/test/java/com/datastax/driver/mapping/MapperAccessorParamsTest.java @@ -21,6 +21,7 @@ import com.datastax.driver.core.Row; import com.datastax.driver.core.exceptions.CodecNotFoundException; import com.datastax.driver.core.utils.CassandraVersion; +import com.datastax.driver.core.utils.ScyllaSkip; import com.datastax.driver.mapping.annotations.Accessor; import com.datastax.driver.mapping.annotations.Param; import com.datastax.driver.mapping.annotations.PartitionKey; @@ -44,6 +45,7 @@ public void onTestContextInitialized() { @Test(groups = "short") @CassandraVersion(value = "2.0", description = "Uses named parameters") + @ScyllaSkip /* @IntegrationTestDisabledScyllaFailure Disabled due to scylladb/scylladb#11945 */ public void should_allow_less_parameters_than_bind_markers_if_there_are_repeated_names() { UserPhoneAccessor accessor = new MappingManager(session()).createAccessor(UserPhoneAccessor.class); From ba62da10c1df27d10901a42f241ced10b4e5dacf Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Wed, 9 Nov 2022 16:42:25 +0100 Subject: [PATCH 2/4] Detect X.Y.0-rcZ semver in version_fetch.py After scylladb/scylla-machine-image#359 was merged, naming of RC versions of Scylla was changed from X.Y.rcZ to X.Y.0-rcZ. version_fetch.py script did not detect those versions. Scylla OSS from 5.1 and Scylla Enterprise from 2022.2 use this new naming scheme. Fix the issue by extending the SCYLLA_OSS_RC_VERSION_REGEX and SCYLLA_ENTERPRISE_RC_VERSION_REGEX regexes. --- ci/version_fetch.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/version_fetch.py b/ci/version_fetch.py index a1eab627dba..7d624e9d40d 100644 --- a/ci/version_fetch.py +++ b/ci/version_fetch.py @@ -25,11 +25,12 @@ SCYLLA_OSS = (DOCKER_HUB_SCYLLA_NAMESPACE, 'scylla') SCYLLA_OSS_RELEASED_VERSION_REGEX = re.compile(r'(\d+)\.(\d+)\.(\d+)') -SCYLLA_OSS_RC_VERSION_REGEX = re.compile(r'(\d+)\.(\d+)\.rc(\d+)') +SCYLLA_OSS_RC_VERSION_REGEX = re.compile(r'(\d+)\.(\d+)\.(?:0-)?rc(\d+)') SCYLLA_ENTERPRISE = (DOCKER_HUB_SCYLLA_NAMESPACE, 'scylla-enterprise') SCYLLA_ENTERPRISE_RELEASED_VERSION_REGEX = re.compile(r'(\d{4})\.(\d+)\.(\d+)') -SCYLLA_ENTERPRISE_RC_VERSION_REGEX = re.compile(r'(\d{4})\.(\d+)\.rc(\d+)') +SCYLLA_ENTERPRISE_RC_VERSION_REGEX = re.compile( + r'(\d{4})\.(\d+)\.(?:0-)?rc(\d+)') CASSANDRA_ENDPOINT = 'https://dlcdn.apache.org/cassandra/' @@ -87,7 +88,7 @@ def fetch_all_scylla_oss_rc_versions(): # Download Docker tags for repository tags_data = fetch_docker_hub_tags(*SCYLLA_OSS) - # Parse only those tags which match 'NUM.NUM.rcNUM' + # Parse only those tags which match 'NUM.NUM.rcNUM' or 'NUM.NUM.0-rcNUM' # into tuple (NUM, NUM, NUM) rc_tags_data = filter(SCYLLA_OSS_RC_VERSION_REGEX.fullmatch, tags_data) rc_tags_data = map(lambda e: SCYLLA_OSS_RC_VERSION_REGEX.match( @@ -112,7 +113,9 @@ def fetch_all_scylla_oss_rc_versions(): # Filter out those RCs that are obsoleted by released stable version rc_tags_data = filter(lambda e: ( e[0], e[1]) not in stable_tags_data, rc_tags_data) - rc_tags_data = [f'{e[0]}.{e[1]}.rc{e[2]}' for e in rc_tags_data] + rc_tags_data = [ + f'{e[0]}.{e[1]}.0-rc{e[2]}' if (e[0], e[1]) >= (5, 1) else + f'{e[0]}.{e[1]}.rc{e[2]}' for e in rc_tags_data] return rc_tags_data @@ -144,7 +147,7 @@ def fetch_all_scylla_enterprise_rc_versions(): # Download Docker tags for repository tags_data = fetch_docker_hub_tags(*SCYLLA_ENTERPRISE) - # Parse only those tags which match 'YEAR.NUM.rcNUM' + # Parse only those tags which match 'YEAR.NUM.rcNUM' or 'YEAR.NUM.0-rcNUM' # into tuple (YEAR, NUM, NUM) rc_tags_data = filter( SCYLLA_ENTERPRISE_RC_VERSION_REGEX.fullmatch, tags_data) @@ -169,7 +172,8 @@ def fetch_all_scylla_enterprise_rc_versions(): # Filter out those RCs that are obsoleted by released stable version rc_tags_data = filter(lambda e: ( e[0], e[1]) not in stable_tags_data, rc_tags_data) - rc_tags_data = [f'{e[0]}.{e[1]}.rc{e[2]}' for e in rc_tags_data] + rc_tags_data = [f'{e[0]}.{e[1]}.0-rc{e[2]}' if (e[0], e[1]) >= + (2022, 2) else f'{e[0]}.{e[1]}.rc{e[2]}' for e in rc_tags_data] return rc_tags_data From f8793f96aa10cce40d7d28a7bf914e4d838a518e Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Wed, 9 Nov 2022 23:29:04 +0100 Subject: [PATCH 3/4] Fix detection of non-obsolete RC Enterprise tags Before the change, stable_tags_data was an iterator, but it could be read multiple times. In such a case, the second time it would return an empty result. Fix the problem by constructing a set. This was a copy-paste error from fetch_all_scylla_oss_rc_versions(). --- ci/version_fetch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/version_fetch.py b/ci/version_fetch.py index 7d624e9d40d..c9a5d4c5bff 100644 --- a/ci/version_fetch.py +++ b/ci/version_fetch.py @@ -162,6 +162,7 @@ def fetch_all_scylla_enterprise_rc_versions(): stable_tags_data = map(lambda e: SCYLLA_ENTERPRISE_RELEASED_VERSION_REGEX.match( e).groups(), stable_tags_data) stable_tags_data = map(lambda e: tuple(map(int, e[0:2])), stable_tags_data) + stable_tags_data = set(stable_tags_data) # Group by (major, minor) and select latest RC version rc_tags_data = sorted(rc_tags_data) From d75744b8eb95f9f8fcf2779d3f1c9452c255aeb0 Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Thu, 10 Nov 2022 15:58:54 +0100 Subject: [PATCH 4/4] Connect to server specified in serverless bundle Before this change, the driver, when connecting to a serverless cluster, would try to resolve host-id.* domains as a proxy IP for each node. This commit aligns the behavior with scylladb/python-driver#177 and scylladb/gocql#106. Now, the "Server" field of bundle is the single hostname/IP the driver will use for proxy connection. This simplifies the setup we had in integration tests, removing the necessity of hacks to work around previous behavior (driver trying to resolve host-id.* domains). Fixes #164 Fixes #167 Fixes #169 --- .../com/datastax/driver/core/Cluster.java | 4 +- .../core/ScyllaCloudSniEndPointFactory.java | 8 ++-- .../datastax/driver/core/CCMTestsSupport.java | 23 +---------- .../driver/core/MockSniEndPointFactory.java | 39 ------------------- 4 files changed, 7 insertions(+), 67 deletions(-) delete mode 100644 driver-core/src/test/java/com/datastax/driver/core/MockSniEndPointFactory.java diff --git a/driver-core/src/main/java/com/datastax/driver/core/Cluster.java b/driver-core/src/main/java/com/datastax/driver/core/Cluster.java index bf8e02fd295..d6c5489f0f7 100644 --- a/driver-core/src/main/java/com/datastax/driver/core/Cluster.java +++ b/driver-core/src/main/java/com/datastax/driver/core/Cluster.java @@ -1406,7 +1406,7 @@ protected Builder withScyllaCloudConnectionConfig(ScyllaCloudConnectionConfig co Builder builder = withEndPointFactory( new ScyllaCloudSniEndPointFactory( - currentDatacenter.getNodeDomain(), proxyAddress.getPort())) + proxyAddress, currentDatacenter.getNodeDomain())) .withSSL( (config.getCurrentAuthInfo().isInsecureSkipTlsVerify() ? config.createBundle().getInsecureSSLOptions() @@ -1423,7 +1423,7 @@ protected Builder withScyllaCloudConnectionConfig(ScyllaCloudConnectionConfig co throw new IllegalStateException( "Can't use withCloudSecureConnectBundle if you've already called addContactPoint(s)"); } - builder.addContactPoint(new SniEndPoint(proxyAddress, proxyAddress.getHostName())); + builder.addContactPoint(new SniEndPoint(proxyAddress, currentDatacenter.getNodeDomain())); return builder; } catch (IOException e) { diff --git a/driver-core/src/main/java/com/datastax/driver/core/ScyllaCloudSniEndPointFactory.java b/driver-core/src/main/java/com/datastax/driver/core/ScyllaCloudSniEndPointFactory.java index a373dd9abe5..81a009b4a05 100644 --- a/driver-core/src/main/java/com/datastax/driver/core/ScyllaCloudSniEndPointFactory.java +++ b/driver-core/src/main/java/com/datastax/driver/core/ScyllaCloudSniEndPointFactory.java @@ -25,11 +25,12 @@ class ScyllaCloudSniEndPointFactory implements EndPointFactory { private final String nodeDomain; - private final int port; - public ScyllaCloudSniEndPointFactory(String nodeDomain, int port) { + private final InetSocketAddress proxy; + + public ScyllaCloudSniEndPointFactory(InetSocketAddress proxy, String nodeDomain) { + this.proxy = proxy; this.nodeDomain = nodeDomain; - this.port = port; } @Override @@ -39,7 +40,6 @@ public void init(Cluster cluster) {} public EndPoint create(Row row) { UUID host_id = row.getUUID("host_id"); String sni = host_id.toString() + "." + nodeDomain; - InetSocketAddress proxy = InetSocketAddress.createUnresolved(sni, port); return new SniEndPoint(proxy, sni); } } diff --git a/driver-core/src/test/java/com/datastax/driver/core/CCMTestsSupport.java b/driver-core/src/test/java/com/datastax/driver/core/CCMTestsSupport.java index 3b0f0e9f88d..6200fc2338b 100644 --- a/driver-core/src/test/java/com/datastax/driver/core/CCMTestsSupport.java +++ b/driver-core/src/test/java/com/datastax/driver/core/CCMTestsSupport.java @@ -24,7 +24,6 @@ import static com.datastax.driver.core.CreateCCM.TestMode.PER_CLASS; import static com.datastax.driver.core.CreateCCM.TestMode.PER_METHOD; import static com.datastax.driver.core.TestUtils.CREATE_KEYSPACE_SIMPLE_FORMAT; -import static com.datastax.driver.core.TestUtils.addressOfNode; import static com.datastax.driver.core.TestUtils.executeNoFail; import static com.datastax.driver.core.TestUtils.ipOfNode; import static org.assertj.core.api.Assertions.fail; @@ -40,7 +39,6 @@ import com.google.common.util.concurrent.Uninterruptibles; import java.io.Closeable; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.Constructor; @@ -778,26 +776,7 @@ public Cluster.Builder createClusterBuilderScyllaCloud() throws IOException { File clusterFile = new File(ccmdir, ccm.getClusterName()); File yamlFile = new File(clusterFile, "config_data.yaml"); - final ScyllaCloudConnectionConfig cloudConfig = - ScyllaCloudConnectionConfig.fromInputStream(new FileInputStream(yamlFile)); - - builder.withScyllaCloudConnectionConfig(cloudConfig); - builder.withEndPointFactory( - new MockSniEndPointFactory( - InetSocketAddress.createUnresolved( - addressOfNode(1).getHostAddress(), - cloudConfig.getCurrentDatacenter().getServer().getPort()), - builder.getConfiguration().getPolicies().getEndPointFactory())); - builder.addContactPoint( - new SniEndPoint( - InetSocketAddress.createUnresolved( - addressOfNode(1).getHostAddress(), - cloudConfig.getCurrentDatacenter().getServer().getPort()), - cloudConfig.getCurrentDatacenter().getServer().getHostName())); - - builder - .withCodecRegistry(new CodecRegistry()) - .withPort(cloudConfig.getCurrentDatacenter().getServer().getPort()); + builder.withScyllaCloudConnectionConfig(yamlFile); return builder; } diff --git a/driver-core/src/test/java/com/datastax/driver/core/MockSniEndPointFactory.java b/driver-core/src/test/java/com/datastax/driver/core/MockSniEndPointFactory.java deleted file mode 100644 index 9ac70436820..00000000000 --- a/driver-core/src/test/java/com/datastax/driver/core/MockSniEndPointFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2022 ScyllaDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.driver.core; - -import java.net.InetSocketAddress; - -class MockSniEndPointFactory implements EndPointFactory { - private final InetSocketAddress proxyAddress; - private final EndPointFactory childSniFactory; - - public MockSniEndPointFactory(InetSocketAddress proxyAddress, EndPointFactory childSniFactory) { - this.proxyAddress = proxyAddress; - this.childSniFactory = childSniFactory; - } - - @Override - public void init(Cluster cluster) { - childSniFactory.init(cluster); - } - - @Override - public EndPoint create(Row peersRow) { - SniEndPoint originalEndPoint = (SniEndPoint) childSniFactory.create(peersRow); - return new SniEndPoint(proxyAddress, originalEndPoint.getServerName()); - } -}