From ec2c8565f14bdcb0a1226a22428546ca03294f55 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 29 Nov 2023 16:14:01 +0100 Subject: [PATCH 1/3] cluster resilience tests --- .github/workflows/resilience.yml | 16 +- .../src/test/java/resilience/ClusterTest.java | 34 +- .../src/test/java/resilience/Endpoint.java | 20 + .../java/resilience/SingleServerTest.java | 20 +- .../connection/ConnectionClusterTest.java | 188 +++++++++ .../resilience/connection/ConnectionTest.java | 10 +- .../resilience/retry/RetryClusterTest.java | 364 ++++++++++++++++++ .../test/java/resilience/retry/RetryTest.java | 32 +- .../shutdown/ShutdownClusterTest.java | 94 +++++ .../timeout/TimeoutClusterTest.java | 116 ++++++ .../java/resilience/utils/MemoryAppender.java | 9 +- .../vstKeepAlive/VstKeepAliveCloseTest.java | 12 +- 12 files changed, 852 insertions(+), 63 deletions(-) create mode 100644 resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java create mode 100644 resilience-tests/src/test/java/resilience/retry/RetryClusterTest.java create mode 100644 resilience-tests/src/test/java/resilience/shutdown/ShutdownClusterTest.java create mode 100644 resilience-tests/src/test/java/resilience/timeout/TimeoutClusterTest.java diff --git a/.github/workflows/resilience.yml b/.github/workflows/resilience.yml index e5f5a8302..06703ed26 100644 --- a/.github/workflows/resilience.yml +++ b/.github/workflows/resilience.yml @@ -3,7 +3,19 @@ name: Resilience Tests on: workflow_dispatch: push: - tags: [ v** ] + branches: + - main + - devel + paths-ignore: + - 'docker/**' + - 'tutorial/**' + - 'ChangeLog.md' + - 'README.md' + pull_request: + types: [ opened, synchronize, reopened ] + branches: + - main + jobs: test: @@ -26,6 +38,8 @@ jobs: cache: maven - name: Start Database run: ./docker/start_db.sh + env: + STARTER_MODE: cluster - name: Info run: mvn -version - name: Start Toxiproxy diff --git a/resilience-tests/src/test/java/resilience/ClusterTest.java b/resilience-tests/src/test/java/resilience/ClusterTest.java index 65ad917df..d23ff45a2 100644 --- a/resilience-tests/src/test/java/resilience/ClusterTest.java +++ b/resilience-tests/src/test/java/resilience/ClusterTest.java @@ -28,26 +28,26 @@ public abstract class ClusterTest { @BeforeAll static void beforeAll() throws IOException { ToxiproxyClient client = new ToxiproxyClient(HOST, 8474); - for (Endpoint ph : endpoints) { - Proxy p = client.getProxyOrNull(ph.getName()); + for (Endpoint endpoint : endpoints) { + Proxy p = client.getProxyOrNull(endpoint.getName()); if (p != null) { p.delete(); } - ph.setProxy(client.createProxy(ph.getName(), ph.getHost() + ":" + ph.getPort(), ph.getUpstream())); + endpoint.setProxy(client.createProxy(endpoint.getName(), endpoint.getHost() + ":" + endpoint.getPort(), endpoint.getUpstream())); } } @AfterAll static void afterAll() throws IOException { - for (Endpoint ph : endpoints) { - ph.getProxy().delete(); + for (Endpoint endpoint : endpoints) { + endpoint.getProxy().delete(); } } @BeforeEach void beforeEach() throws IOException { - for (Endpoint ph : endpoints) { - ph.getProxy().enable(); + for (Endpoint endpoint : endpoints) { + endpoint.getProxy().enable(); } } @@ -56,11 +56,23 @@ protected static List getEndpoints() { } protected static ArangoDB.Builder dbBuilder() { - ArangoDB.Builder builder = new ArangoDB.Builder().password(PASSWORD); - for (Endpoint ph : endpoints) { - builder.host(ph.getHost(), ph.getPort()); + ArangoDB.Builder builder = new ArangoDB.Builder(); + for (Endpoint endpoint : endpoints) { + builder.host(endpoint.getHost(), endpoint.getPort()); + } + return builder.password(PASSWORD); + } + + protected void enableAllEndpoints(){ + for (Endpoint endpoint : endpoints) { + endpoint.enable(); + } + } + + protected void disableAllEndpoints(){ + for (Endpoint endpoint : endpoints) { + endpoint.disable(); } - return builder; } } diff --git a/resilience-tests/src/test/java/resilience/Endpoint.java b/resilience-tests/src/test/java/resilience/Endpoint.java index 9e8c697d4..8c633725e 100644 --- a/resilience-tests/src/test/java/resilience/Endpoint.java +++ b/resilience-tests/src/test/java/resilience/Endpoint.java @@ -2,6 +2,8 @@ import eu.rekawek.toxiproxy.Proxy; +import java.io.IOException; + /** * class representing a proxied db endpoint */ @@ -42,4 +44,22 @@ public Proxy getProxy() { public void setProxy(Proxy proxy) { this.proxy = proxy; } + + public void enable() { + try { + getProxy().enable(); + Thread.sleep(100); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + public void disable() { + try { + getProxy().disable(); + Thread.sleep(100); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } } diff --git a/resilience-tests/src/test/java/resilience/SingleServerTest.java b/resilience-tests/src/test/java/resilience/SingleServerTest.java index 889376d2e..3aa587d93 100644 --- a/resilience-tests/src/test/java/resilience/SingleServerTest.java +++ b/resilience-tests/src/test/java/resilience/SingleServerTest.java @@ -36,7 +36,7 @@ static void afterAll() throws IOException { @BeforeEach void beforeEach() { - enableEndpoint(); + getEndpoint().enable(); } protected static Endpoint getEndpoint() { @@ -49,22 +49,4 @@ protected static ArangoDB.Builder dbBuilder() { .password(PASSWORD); } - protected void enableEndpoint(){ - try { - getEndpoint().getProxy().enable(); - Thread.sleep(100); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - protected void disableEndpoint(){ - try { - getEndpoint().getProxy().disable(); - Thread.sleep(100); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - } diff --git a/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java b/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java new file mode 100644 index 000000000..ab3985ade --- /dev/null +++ b/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java @@ -0,0 +1,188 @@ +package resilience.connection; + +import ch.qos.logback.classic.Level; +import com.arangodb.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import resilience.ClusterTest; + +import java.net.ConnectException; +import java.net.UnknownHostException; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** + * @author Michele Rastelli + */ +class ConnectionClusterTest extends ClusterTest { + + static Stream protocolProvider() { + return Stream.of( + Protocol.VST, + Protocol.HTTP_VPACK, + Protocol.HTTP2_VPACK + ); + } + + static Stream arangoProvider() { + return Stream.of( + dbBuilder().protocol(Protocol.VST).build(), + dbBuilder().protocol(Protocol.HTTP_VPACK).build(), + dbBuilder().protocol(Protocol.HTTP2_JSON).build() + ); + } + + static Stream asyncArangoProvider() { + return arangoProvider().map(ArangoDB::async); + } + + @ParameterizedTest + @MethodSource("protocolProvider") + void nameResolutionFail(Protocol protocol) { + ArangoDB arangoDB = new ArangoDB.Builder() + .host("wrongHost", 8529) + .protocol(protocol) + .build(); + + Throwable thrown = catchThrowable(arangoDB::getVersion); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host!"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> { + assertThat(e).isInstanceOf(UnknownHostException.class); + assertThat(e.getMessage()).contains("wrongHost"); + }); + arangoDB.shutdown(); + } + + @ParameterizedTest + @MethodSource("protocolProvider") + void nameResolutionFailAsync(Protocol protocol) { + ArangoDBAsync arangoDB = new ArangoDB.Builder() + .host("wrongHost", 8529) + .protocol(protocol) + .build() + .async(); + + Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host!"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> { + assertThat(e).isInstanceOf(UnknownHostException.class); + assertThat(e.getMessage()).contains("wrongHost"); + }); + arangoDB.shutdown(); + } + + @ParameterizedTest + @MethodSource("protocolProvider") + void nameResolutionFailover(Protocol protocol) { + ArangoDB arangoDB = new ArangoDB.Builder() + .password("test") + .host("wrongHost", 8529) + .host("127.0.0.1", 8529) + .protocol(protocol) + .build(); + + arangoDB.getVersion(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + } + + @ParameterizedTest + @MethodSource("protocolProvider") + void nameResolutionFailoverAsync(Protocol protocol) throws ExecutionException, InterruptedException { + ArangoDBAsync arangoDB = new ArangoDB.Builder() + .password("test") + .host("wrongHost", 8529) + .host("127.0.0.1", 8529) + .protocol(protocol) + .build() + .async(); + + arangoDB.getVersion().get(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void connectionFail(ArangoDB arangoDB) { + disableAllEndpoints(); + + Throwable thrown = catchThrowable(arangoDB::getVersion); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> + assertThat(e).isInstanceOf(ConnectException.class)); + + arangoDB.shutdown(); + enableAllEndpoints(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void connectionFailAsync(ArangoDBAsync arangoDB) { + disableAllEndpoints(); + + Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> + assertThat(e).isInstanceOf(ConnectException.class)); + arangoDB.shutdown(); + enableAllEndpoints(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void connectionFailover(ArangoDB arangoDB) { + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.getVersion(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + enableAllEndpoints(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void connectionFailoverAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException { + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.getVersion().get(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + enableAllEndpoints(); + } + + +} diff --git a/resilience-tests/src/test/java/resilience/connection/ConnectionTest.java b/resilience-tests/src/test/java/resilience/connection/ConnectionTest.java index f41ff9b9d..4c16ad6ce 100644 --- a/resilience-tests/src/test/java/resilience/connection/ConnectionTest.java +++ b/resilience-tests/src/test/java/resilience/connection/ConnectionTest.java @@ -1,9 +1,9 @@ package resilience.connection; import com.arangodb.*; -import resilience.SingleServerTest; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import resilience.SingleServerTest; import java.net.ConnectException; import java.net.UnknownHostException; @@ -81,7 +81,7 @@ void nameResolutionFailAsync(Protocol protocol) { @ParameterizedTest(name = "{index}") @MethodSource("arangoProvider") void connectionFail(ArangoDB arangoDB) { - disableEndpoint(); + getEndpoint().disable(); Throwable thrown = catchThrowable(arangoDB::getVersion); assertThat(thrown).isInstanceOf(ArangoDBException.class); @@ -91,13 +91,13 @@ void connectionFail(ArangoDB arangoDB) { ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> assertThat(e).isInstanceOf(ConnectException.class)); arangoDB.shutdown(); - enableEndpoint(); + getEndpoint().enable(); } @ParameterizedTest(name = "{index}") @MethodSource("asyncArangoProvider") void connectionFailAsync(ArangoDBAsync arangoDB) { - disableEndpoint(); + getEndpoint().disable(); Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); assertThat(thrown).isInstanceOf(ArangoDBException.class); @@ -107,7 +107,7 @@ void connectionFailAsync(ArangoDBAsync arangoDB) { ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> assertThat(e).isInstanceOf(ConnectException.class)); arangoDB.shutdown(); - enableEndpoint(); + getEndpoint().enable(); } } diff --git a/resilience-tests/src/test/java/resilience/retry/RetryClusterTest.java b/resilience-tests/src/test/java/resilience/retry/RetryClusterTest.java new file mode 100644 index 000000000..ff07f37d4 --- /dev/null +++ b/resilience-tests/src/test/java/resilience/retry/RetryClusterTest.java @@ -0,0 +1,364 @@ +package resilience.retry; + +import ch.qos.logback.classic.Level; +import com.arangodb.*; +import eu.rekawek.toxiproxy.model.ToxicDirection; +import eu.rekawek.toxiproxy.model.toxic.Latency; +import io.vertx.core.http.HttpClosedException; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import resilience.ClusterTest; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.concurrent.*; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * @author Michele Rastelli + */ +class RetryClusterTest extends ClusterTest { + + static Stream arangoProvider() { + return Stream.of( + dbBuilder().protocol(Protocol.VST).build(), + dbBuilder().protocol(Protocol.HTTP_VPACK).build(), + dbBuilder().protocol(Protocol.HTTP2_VPACK).build() + ); + } + + static Stream asyncArangoProvider() { + return arangoProvider().map(ArangoDB::async); + } + + /** + * on reconnection failure: - 3x logs WARN Could not connect to host[addr=127.0.0.1,port=8529] - + * ArangoDBException("Cannot contact any host") + *

+ * once the proxy is re-enabled: - the subsequent requests should be successful + */ + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void unreachableHost(ArangoDB arangoDB) { + arangoDB.getVersion(); + disableAllEndpoints(); + + for (int i = 0; i < 10; i++) { + Throwable thrown = catchThrowable(arangoDB::getVersion); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> + assertThat(e).isInstanceOf(ConnectException.class)); + } + + long warnsCount = logs.getLogs() + .filter(e -> e.getLevel().equals(Level.WARN)) + .filter(e -> e.getFormattedMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) + .count(); + assertThat(warnsCount).isGreaterThanOrEqualTo(3); + + enableAllEndpoints(); + arangoDB.getVersion(); + arangoDB.shutdown(); + } + + /** + * on reconnection failure: - 3x logs WARN Could not connect to host[addr=127.0.0.1,port=8529] - + * ArangoDBException("Cannot contact any host") + *

+ * once the proxy is re-enabled: - the subsequent requests should be successful + */ + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void unreachableHostAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException { + arangoDB.getVersion().get(); + disableAllEndpoints(); + + for (int i = 0; i < 10; i++) { + Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("Cannot contact any host"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class); + ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> + assertThat(e).isInstanceOf(ConnectException.class)); + } + + long warnsCount = logs.getLogs() + .filter(e -> e.getLevel().equals(Level.WARN)) + .filter(e -> e.getFormattedMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) + .count(); + assertThat(warnsCount).isGreaterThanOrEqualTo(3); + + enableAllEndpoints(); + arangoDB.getVersion().get(); + arangoDB.shutdown(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void unreachableHostFailover(ArangoDB arangoDB) { + arangoDB.getVersion(); + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.getVersion(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + enableAllEndpoints(); + arangoDB.shutdown(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void unreachableHostFailoverAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException { + arangoDB.getVersion().get(); + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.getVersion().get(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + enableAllEndpoints(); + arangoDB.shutdown(); + } + + /** + * on delayed response: + * - ArangoDBException with cause TimeoutException + *

+ * once the delay is removed: + * - the subsequent requests should be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void connectionTimeout(Protocol protocol) throws IOException, InterruptedException { + // https://github.com/vert-x3/vertx-web/issues/2296 + // WebClient: HTTP/2 request timeout does not throw TimeoutException + assumeTrue(protocol != Protocol.HTTP2_VPACK); + assumeTrue(protocol != Protocol.HTTP2_JSON); + + ArangoDB arangoDB = dbBuilder() + .timeout(1_000) + .protocol(protocol) + .build(); + + arangoDB.getVersion(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + // no failover for TimeoutException + for (int i = 0; i < 2; i++) { + Throwable thrown = catchThrowable(arangoDB::getVersion); + thrown.printStackTrace(); + assertThat(thrown) + .isInstanceOf(ArangoDBException.class) + .extracting(Throwable::getCause) + .isInstanceOf(TimeoutException.class); + } + + toxic.remove(); + Thread.sleep(100); + + arangoDB.getVersion(); + arangoDB.shutdown(); + } + + /** + * on delayed response: + * - ArangoDBException with cause TimeoutException + *

+ * once the delay is removed: + * - the subsequent requests should be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void connectionTimeoutAsync(Protocol protocol) throws IOException, InterruptedException, ExecutionException { + // https://github.com/vert-x3/vertx-web/issues/2296 + // WebClient: HTTP/2 request timeout does not throw TimeoutException + assumeTrue(protocol != Protocol.HTTP2_VPACK); + assumeTrue(protocol != Protocol.HTTP2_JSON); + + ArangoDBAsync arangoDB = dbBuilder() + .timeout(1_000) + .protocol(protocol) + .build() + .async(); + + arangoDB.getVersion().get(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + // no failover for TimeoutException + for (int i = 0; i < 2; i++) { + Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); + thrown.printStackTrace(); + assertThat(thrown) + .isInstanceOf(ArangoDBException.class) + .extracting(Throwable::getCause) + .isInstanceOf(TimeoutException.class); + } + + toxic.remove(); + Thread.sleep(100); + + arangoDB.getVersion().get(); + arangoDB.shutdown(); + } + + + @ParameterizedTest + @EnumSource(Protocol.class) + void retryGetOnClosedConnection(Protocol protocol) throws IOException, InterruptedException { + assumeTrue(protocol != Protocol.VST); + ArangoDB arangoDB = dbBuilder() + .protocol(protocol) + .build(); + + arangoDB.getVersion(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + arangoDB.getVersion(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + toxic.remove(); + enableAllEndpoints(); + arangoDB.shutdown(); + es.shutdown(); + } + + @ParameterizedTest + @EnumSource(Protocol.class) + void retryGetOnClosedConnectionAsync(Protocol protocol) throws IOException, InterruptedException, ExecutionException { + assumeTrue(protocol != Protocol.VST); + ArangoDBAsync arangoDB = dbBuilder() + .protocol(protocol) + .build() + .async(); + + arangoDB.getVersion().get(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + arangoDB.getVersion().get(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + toxic.remove(); + enableAllEndpoints(); + arangoDB.shutdown(); + es.shutdown(); + } + + + /** + * on closed pending requests of unsafe HTTP methods: - no retry should happen + *

+ * the subsequent requests should fail over to a different coordinator and be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void notRetryPostOnClosedConnection(Protocol protocol) throws IOException, InterruptedException { + ArangoDB arangoDB = dbBuilder() + .protocol(protocol) + .build(); + + arangoDB.db().query("return null", Void.class); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + Throwable thrown = catchThrowable(() -> arangoDB.db().query("return null", Void.class)); + thrown.printStackTrace(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + if (protocol != Protocol.VST) { + assertThat(thrown.getCause().getCause()).isInstanceOf(HttpClosedException.class); + } + + arangoDB.db().query("return null", Void.class); + + toxic.remove(); + enableAllEndpoints(); + + arangoDB.shutdown(); + es.shutdown(); + } + + /** + * on closed pending requests of unsafe HTTP methods: - no retry should happen + *

+ * the subsequent requests should fail over to a different coordinator and be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void notRetryPostOnClosedConnectionAsync(Protocol protocol) throws IOException, InterruptedException, ExecutionException { + ArangoDBAsync arangoDB = dbBuilder() + .protocol(protocol) + .build() + .async(); + + arangoDB.db().query("return null", Void.class).get(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + Throwable thrown = catchThrowable(() -> arangoDB.db().query("return null", Void.class).get()).getCause(); + thrown.printStackTrace(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + if (protocol != Protocol.VST) { + assertThat(thrown.getCause().getCause()).isInstanceOf(HttpClosedException.class); + } + + arangoDB.db().query("return null", Void.class).get(); + + toxic.remove(); + enableAllEndpoints(); + + arangoDB.shutdown(); + es.shutdown(); + } + +} diff --git a/resilience-tests/src/test/java/resilience/retry/RetryTest.java b/resilience-tests/src/test/java/resilience/retry/RetryTest.java index 6a7fc7d38..df2d5cd72 100644 --- a/resilience-tests/src/test/java/resilience/retry/RetryTest.java +++ b/resilience-tests/src/test/java/resilience/retry/RetryTest.java @@ -47,7 +47,7 @@ static Stream asyncArangoProvider() { @MethodSource("arangoProvider") void unreachableHost(ArangoDB arangoDB) { arangoDB.getVersion(); - disableEndpoint(); + getEndpoint().disable(); for (int i = 0; i < 10; i++) { Throwable thrown = catchThrowable(arangoDB::getVersion); @@ -59,13 +59,13 @@ void unreachableHost(ArangoDB arangoDB) { assertThat(e).isInstanceOf(ConnectException.class)); } - long warnsCount = logs.getLoggedEvents().stream() + long warnsCount = logs.getLogs() .filter(e -> e.getLevel().equals(Level.WARN)) - .filter(e -> e.getMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) + .filter(e -> e.getFormattedMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) .count(); assertThat(warnsCount).isGreaterThanOrEqualTo(3); - enableEndpoint(); + getEndpoint().enable(); arangoDB.getVersion(); arangoDB.shutdown(); } @@ -80,7 +80,7 @@ void unreachableHost(ArangoDB arangoDB) { @MethodSource("asyncArangoProvider") void unreachableHostAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException { arangoDB.getVersion().get(); - disableEndpoint(); + getEndpoint().disable(); for (int i = 0; i < 10; i++) { Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); @@ -92,13 +92,13 @@ void unreachableHostAsync(ArangoDBAsync arangoDB) throws ExecutionException, Int assertThat(e).isInstanceOf(ConnectException.class)); } - long warnsCount = logs.getLoggedEvents().stream() + long warnsCount = logs.getLogs() .filter(e -> e.getLevel().equals(Level.WARN)) - .filter(e -> e.getMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) + .filter(e -> e.getFormattedMessage().contains("Could not connect to host[addr=127.0.0.1,port=18529]")) .count(); assertThat(warnsCount).isGreaterThanOrEqualTo(3); - enableEndpoint(); + getEndpoint().enable(); arangoDB.getVersion().get(); arangoDB.shutdown(); } @@ -210,7 +210,7 @@ void retryGetOnClosedConnection(Protocol protocol) throws IOException, Interrupt Thread.sleep(100); ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); - es.schedule(this::disableEndpoint, 300, TimeUnit.MILLISECONDS); + es.schedule(() -> getEndpoint().disable(), 300, TimeUnit.MILLISECONDS); Throwable thrown = catchThrowable(arangoDB::getVersion); thrown.printStackTrace(); @@ -225,7 +225,7 @@ void retryGetOnClosedConnection(Protocol protocol) throws IOException, Interrupt toxic.remove(); Thread.sleep(100); - enableEndpoint(); + getEndpoint().enable(); arangoDB.getVersion(); arangoDB.shutdown(); @@ -258,7 +258,7 @@ void retryGetOnClosedConnectionAsync(Protocol protocol) throws IOException, Inte Thread.sleep(100); ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); - es.schedule(this::disableEndpoint, 300, TimeUnit.MILLISECONDS); + es.schedule(() -> getEndpoint().disable(), 300, TimeUnit.MILLISECONDS); Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); thrown.printStackTrace(); @@ -273,7 +273,7 @@ void retryGetOnClosedConnectionAsync(Protocol protocol) throws IOException, Inte toxic.remove(); Thread.sleep(100); - enableEndpoint(); + getEndpoint().enable(); arangoDB.getVersion().get(); arangoDB.shutdown(); @@ -300,7 +300,7 @@ void notRetryPostOnClosedConnection(Protocol protocol) throws IOException, Inter Thread.sleep(100); ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); - es.schedule(this::disableEndpoint, 300, TimeUnit.MILLISECONDS); + es.schedule(() -> getEndpoint().disable(), 300, TimeUnit.MILLISECONDS); Throwable thrown = catchThrowable(() -> arangoDB.db().query("return null", Void.class)); thrown.printStackTrace(); @@ -312,7 +312,7 @@ void notRetryPostOnClosedConnection(Protocol protocol) throws IOException, Inter toxic.remove(); Thread.sleep(100); - enableEndpoint(); + getEndpoint().enable(); arangoDB.db().query("return null", Void.class); arangoDB.shutdown(); @@ -339,7 +339,7 @@ void notRetryPostOnClosedConnectionAsync(Protocol protocol) throws IOException, Thread.sleep(100); ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); - es.schedule(this::disableEndpoint, 300, TimeUnit.MILLISECONDS); + es.schedule(() -> getEndpoint().disable(), 300, TimeUnit.MILLISECONDS); Throwable thrown = catchThrowable(() -> arangoDB.db().query("return null", Void.class).get()).getCause(); thrown.printStackTrace(); @@ -351,7 +351,7 @@ void notRetryPostOnClosedConnectionAsync(Protocol protocol) throws IOException, toxic.remove(); Thread.sleep(100); - enableEndpoint(); + getEndpoint().enable(); arangoDB.db().query("return null", Void.class).get(); arangoDB.shutdown(); diff --git a/resilience-tests/src/test/java/resilience/shutdown/ShutdownClusterTest.java b/resilience-tests/src/test/java/resilience/shutdown/ShutdownClusterTest.java new file mode 100644 index 000000000..082cb298e --- /dev/null +++ b/resilience-tests/src/test/java/resilience/shutdown/ShutdownClusterTest.java @@ -0,0 +1,94 @@ +package resilience.shutdown; + +import com.arangodb.ArangoDB; +import com.arangodb.ArangoDBAsync; +import com.arangodb.ArangoDBException; +import com.arangodb.Protocol; +import io.vertx.core.http.HttpClosedException; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import resilience.ClusterTest; +import resilience.SingleServerTest; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * @author Michele Rastelli + */ +class ShutdownClusterTest extends ClusterTest { + + @ParameterizedTest + @EnumSource(Protocol.class) + void shutdown(Protocol protocol) throws InterruptedException { + ArangoDB arangoDB = dbBuilder() + .protocol(protocol) + .build(); + + arangoDB.getVersion(); + arangoDB.shutdown(); + Thread.sleep(1_000); + Throwable thrown = catchThrowable(arangoDB::getVersion); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("closed"); + } + + @ParameterizedTest + @EnumSource(Protocol.class) + void shutdownAsync(Protocol protocol) throws InterruptedException, ExecutionException { + ArangoDBAsync arangoDB = dbBuilder() + .protocol(protocol) + .build() + .async(); + + arangoDB.getVersion().get(); + arangoDB.shutdown(); + Thread.sleep(1_000); + Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getMessage()).contains("closed"); + } + + @ParameterizedTest + @EnumSource(Protocol.class) + void shutdownWithPendingRequests(Protocol protocol) { + assumeTrue(protocol != Protocol.VST); + ArangoDB arangoDB = dbBuilder() + .protocol(protocol) + .build(); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(arangoDB::shutdown, 500, TimeUnit.MILLISECONDS); + Throwable thrown = catchThrowable(() -> arangoDB.db().query("return sleep(1)", Void.class)); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + assertThat(thrown.getCause().getCause()).isInstanceOf(HttpClosedException.class); + es.shutdown(); + } + + @ParameterizedTest + @EnumSource(Protocol.class) + void shutdownWithPendingRequestsAsync(Protocol protocol) { + assumeTrue(protocol != Protocol.VST); + ArangoDBAsync arangoDB = dbBuilder() + .protocol(protocol) + .build() + .async(); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(arangoDB::shutdown, 500, TimeUnit.MILLISECONDS); + Throwable thrown = catchThrowable(() -> arangoDB.db().query("return sleep(1)", Void.class).get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + assertThat(thrown.getCause().getCause()).isInstanceOf(HttpClosedException.class); + es.shutdown(); + } + +} diff --git a/resilience-tests/src/test/java/resilience/timeout/TimeoutClusterTest.java b/resilience-tests/src/test/java/resilience/timeout/TimeoutClusterTest.java new file mode 100644 index 000000000..f5480b607 --- /dev/null +++ b/resilience-tests/src/test/java/resilience/timeout/TimeoutClusterTest.java @@ -0,0 +1,116 @@ +package resilience.timeout; + +import com.arangodb.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import resilience.ClusterTest; +import resilience.SingleServerTest; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * @author Michele Rastelli + */ +class TimeoutClusterTest extends ClusterTest { + + /** + * on timeout failure: + * - throw exception + * - expect operation performed (at most) once + *

+ * after the exception: + * - the subsequent requests should be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void requestTimeout(Protocol protocol) throws InterruptedException { + // https://github.com/vert-x3/vertx-web/issues/2296 + // WebClient: HTTP/2 request timeout does not throw TimeoutException + assumeTrue(protocol != Protocol.HTTP2_VPACK); + assumeTrue(protocol != Protocol.HTTP2_JSON); + + ArangoDB arangoDB = dbBuilder() + .timeout(1_000) + .protocol(protocol) + .build(); + + arangoDB.getVersion(); + String colName = "timeoutTest"; + ArangoCollection col = arangoDB.db().collection(colName); + if (!col.exists()) col.create(); + col.truncate(); + + Throwable thrown = catchThrowable(() -> arangoDB.db() + .query("INSERT {value:sleep(2)} INTO @@col RETURN NEW", + Map.class, + Collections.singletonMap("@col", colName)) + ); + + assertThat(thrown) + .isInstanceOf(ArangoDBException.class) + .extracting(Throwable::getCause) + .isInstanceOf(TimeoutException.class); + + arangoDB.getVersion(); + + Thread.sleep(2_000); + assertThat(col.count().getCount()).isEqualTo(1); + + arangoDB.shutdown(); + } + + /** + * on timeout failure: + * - throw exception + * - expect operation performed (at most) once + *

+ * after the exception: + * - the subsequent requests should be successful + */ + @ParameterizedTest + @EnumSource(Protocol.class) + void requestTimeoutAsync(Protocol protocol) throws InterruptedException, ExecutionException { + // https://github.com/vert-x3/vertx-web/issues/2296 + // WebClient: HTTP/2 request timeout does not throw TimeoutException + assumeTrue(protocol != Protocol.HTTP2_VPACK); + assumeTrue(protocol != Protocol.HTTP2_JSON); + + ArangoDBAsync arangoDB = dbBuilder() + .timeout(1_000) + .protocol(protocol) + .build() + .async(); + + arangoDB.getVersion().get(); + String colName = "timeoutTest"; + ArangoCollectionAsync col = arangoDB.db().collection(colName); + if (!col.exists().get()) col.create().get(); + col.truncate().get(); + + Throwable thrown = catchThrowable(() -> arangoDB.db() + .query("INSERT {value:sleep(2)} INTO @@col RETURN NEW", + Map.class, + Collections.singletonMap("@col", colName)).get() + ).getCause(); + + assertThat(thrown) + .isInstanceOf(ArangoDBException.class) + .extracting(Throwable::getCause) + .isInstanceOf(TimeoutException.class); + + arangoDB.getVersion().get(); + + Thread.sleep(2_000); + assertThat(col.count().get().getCount()).isEqualTo(1); + + arangoDB.shutdown(); + } + +} diff --git a/resilience-tests/src/test/java/resilience/utils/MemoryAppender.java b/resilience-tests/src/test/java/resilience/utils/MemoryAppender.java index da7b9fa37..e09e61720 100644 --- a/resilience-tests/src/test/java/resilience/utils/MemoryAppender.java +++ b/resilience-tests/src/test/java/resilience/utils/MemoryAppender.java @@ -6,8 +6,7 @@ import ch.qos.logback.core.read.ListAppender; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.List; +import java.util.stream.Stream; public class MemoryAppender extends ListAppender { @@ -19,10 +18,10 @@ public MemoryAppender() { } public void reset() { - this.list.clear(); + list.clear(); } - public List getLoggedEvents() { - return Collections.unmodifiableList(this.list); + public Stream getLogs() { + return list.stream(); } } \ No newline at end of file diff --git a/resilience-tests/src/test/java/resilience/vstKeepAlive/VstKeepAliveCloseTest.java b/resilience-tests/src/test/java/resilience/vstKeepAlive/VstKeepAliveCloseTest.java index 7bb52285f..e3d22d12b 100644 --- a/resilience-tests/src/test/java/resilience/vstKeepAlive/VstKeepAliveCloseTest.java +++ b/resilience-tests/src/test/java/resilience/vstKeepAlive/VstKeepAliveCloseTest.java @@ -47,10 +47,10 @@ void shutDown() { void keepAliveCloseAndReconnect() throws IOException { arangoDB.getVersion(); Latency toxic = getEndpoint().getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); - await().until(() -> logs.getLoggedEvents().stream() + await().until(() -> logs.getLogs() .filter(e -> e.getLevel().equals(Level.ERROR)) - .filter(e -> e.getMessage() != null) - .anyMatch(e -> e.getMessage().contains("Connection unresponsive!"))); + .filter(e -> e.getFormattedMessage() != null) + .anyMatch(e -> e.getFormattedMessage().contains("Connection unresponsive!"))); toxic.setLatency(0); toxic.remove(); arangoDB.getVersion(); @@ -66,10 +66,10 @@ void keepAliveCloseAndReconnect() throws IOException { void keepAliveCloseAndReconnectAsync() throws IOException, ExecutionException, InterruptedException { arangoDB.async().getVersion().get(); Latency toxic = getEndpoint().getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); - await().until(() -> logs.getLoggedEvents().stream() + await().until(() -> logs.getLogs() .filter(e -> e.getLevel().equals(Level.ERROR)) - .filter(e -> e.getMessage() != null) - .anyMatch(e -> e.getMessage().contains("Connection unresponsive!"))); + .filter(e -> e.getFormattedMessage() != null) + .anyMatch(e -> e.getFormattedMessage().contains("Connection unresponsive!"))); toxic.setLatency(0); toxic.remove(); arangoDB.async().getVersion().get(); From d5c3717b4e50daf42fbf41112118a8da2e77e541 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 29 Nov 2023 19:15:38 +0100 Subject: [PATCH 2/3] RetriableCursorClusterTest --- .../retry/RetriableCursorClusterTest.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 resilience-tests/src/test/java/resilience/retry/RetriableCursorClusterTest.java diff --git a/resilience-tests/src/test/java/resilience/retry/RetriableCursorClusterTest.java b/resilience-tests/src/test/java/resilience/retry/RetriableCursorClusterTest.java new file mode 100644 index 000000000..6aa443aa5 --- /dev/null +++ b/resilience-tests/src/test/java/resilience/retry/RetriableCursorClusterTest.java @@ -0,0 +1,103 @@ +package resilience.retry; + +import com.arangodb.*; +import com.arangodb.model.AqlQueryOptions; +import eu.rekawek.toxiproxy.model.ToxicDirection; +import eu.rekawek.toxiproxy.model.toxic.Latency; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import resilience.ClusterTest; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** + * @author Michele Rastelli + */ +class RetriableCursorClusterTest extends ClusterTest { + + static Stream arangoProvider() { + return Stream.of( + dbBuilder().timeout(1_000).protocol(Protocol.VST).build(), + dbBuilder().timeout(1_000).protocol(Protocol.HTTP_JSON).build(), + dbBuilder().timeout(1_000).protocol(Protocol.HTTP_VPACK).build() + ); + } + + static Stream asyncArangoProvider() { + return arangoProvider().map(ArangoDB::async); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void retryCursor(ArangoDB arangoDB) throws IOException, InterruptedException { + + ArangoCursor cursor = arangoDB.db() + .query("for i in 1..2 return i", + String.class, + new AqlQueryOptions().batchSize(1).allowRetry(true)); + + assertThat(cursor.hasNext()).isTrue(); + assertThat(cursor.next()).isEqualTo("1"); + assertThat(cursor.hasNext()).isTrue(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + Throwable thrown = catchThrowable(cursor::next); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + + assertThat(cursor.next()).isEqualTo("2"); + assertThat(cursor.hasNext()).isFalse(); + + toxic.remove(); + enableAllEndpoints(); + arangoDB.shutdown(); + es.shutdown(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void retryCursorAsync(ArangoDBAsync arangoDB) throws IOException, InterruptedException, ExecutionException { + + ArangoCursorAsync cursor = arangoDB.db() + .query("for i in 1..2 return i", + String.class, + new AqlQueryOptions().batchSize(1).allowRetry(true)).get(); + + assertThat(cursor.getResult()).containsExactly("1"); + assertThat(cursor.hasMore()).isTrue(); + + // slow down the driver connection + Latency toxic = getEndpoints().get(0).getProxy().toxics().latency("latency", ToxicDirection.DOWNSTREAM, 10_000); + Thread.sleep(100); + + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.schedule(() -> getEndpoints().get(0).disable(), 300, TimeUnit.MILLISECONDS); + + Throwable thrown = catchThrowable(() -> cursor.nextBatch().get()).getCause(); + assertThat(thrown).isInstanceOf(ArangoDBException.class); + assertThat(thrown.getCause()).isInstanceOf(IOException.class); + + ArangoCursorAsync c2 = cursor.nextBatch().get(); + assertThat(c2.getResult()).containsExactly("2"); + assertThat(c2.hasMore()).isFalse(); + + toxic.remove(); + enableAllEndpoints(); + arangoDB.shutdown(); + es.shutdown(); + } +} From 12c6617446815b2d6630bd541f9e6ddf1284d78c Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 29 Nov 2023 19:18:49 +0100 Subject: [PATCH 3/3] connection failover for POST requests --- .../connection/ConnectionClusterTest.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java b/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java index ab3985ade..458285cd9 100644 --- a/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java +++ b/resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java @@ -184,5 +184,36 @@ void connectionFailoverAsync(ArangoDBAsync arangoDB) throws ExecutionException, enableAllEndpoints(); } + @ParameterizedTest(name = "{index}") + @MethodSource("arangoProvider") + void connectionFailoverPost(ArangoDB arangoDB) { + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.db().query("RETURN 1", Integer.class); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + enableAllEndpoints(); + } + + @ParameterizedTest(name = "{index}") + @MethodSource("asyncArangoProvider") + void connectionFailoverPostAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException { + getEndpoints().get(0).disable(); + getEndpoints().get(1).disable(); + + arangoDB.db().query("RETURN 1", Integer.class).get(); + + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + + arangoDB.shutdown(); + enableAllEndpoints(); + } }