[DE-722] cluster resilience tests

.github/workflows/resilience.yml

@@ -3,7 +3,19 @@ name: Resilience Tests
 on:
   workflow_dispatch:
   push:
-    tags: [ v** ]
+    branches:
+      - main
+      - devel
+    paths-ignore:
+      - 'docker/**'
+      - 'tutorial/**'
+      - 'ChangeLog.md'
+      - 'README.md'
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+    branches:
+      - main
+
 
 jobs:
   test:
@@ -26,6 +38,8 @@ jobs:
           cache: maven
       - name: Start Database
         run: ./docker/start_db.sh
+        env:
+          STARTER_MODE: cluster
       - name: Info
         run: mvn -version
       - name: Start Toxiproxy

resilience-tests/src/test/java/resilience/ClusterTest.java

@@ -28,26 +28,26 @@ public abstract class ClusterTest {
     @BeforeAll
     static void beforeAll() throws IOException {
         ToxiproxyClient client = new ToxiproxyClient(HOST, 8474);
-        for (Endpoint ph : endpoints) {
-            Proxy p = client.getProxyOrNull(ph.getName());
+        for (Endpoint endpoint : endpoints) {
+            Proxy p = client.getProxyOrNull(endpoint.getName());
             if (p != null) {
                 p.delete();
             }
-            ph.setProxy(client.createProxy(ph.getName(), ph.getHost() + ":" + ph.getPort(), ph.getUpstream()));
+            endpoint.setProxy(client.createProxy(endpoint.getName(), endpoint.getHost() + ":" + endpoint.getPort(), endpoint.getUpstream()));
         }
     }
 
     @AfterAll
     static void afterAll() throws IOException {
-        for (Endpoint ph : endpoints) {
-            ph.getProxy().delete();
+        for (Endpoint endpoint : endpoints) {
+            endpoint.getProxy().delete();
         }
     }
 
     @BeforeEach
     void beforeEach() throws IOException {
-        for (Endpoint ph : endpoints) {
-            ph.getProxy().enable();
+        for (Endpoint endpoint : endpoints) {
+            endpoint.getProxy().enable();
         }
     }
 
@@ -56,11 +56,23 @@ protected static List<Endpoint> getEndpoints() {
     }
 
     protected static ArangoDB.Builder dbBuilder() {
-        ArangoDB.Builder builder = new ArangoDB.Builder().password(PASSWORD);
-        for (Endpoint ph : endpoints) {
-            builder.host(ph.getHost(), ph.getPort());
+        ArangoDB.Builder builder = new ArangoDB.Builder();
+        for (Endpoint endpoint : endpoints) {
+            builder.host(endpoint.getHost(), endpoint.getPort());
+        }
+        return builder.password(PASSWORD);
+    }
+
+    protected void enableAllEndpoints(){
+        for (Endpoint endpoint : endpoints) {
+            endpoint.enable();
+        }
+    }
+
+    protected void disableAllEndpoints(){
+        for (Endpoint endpoint : endpoints) {
+            endpoint.disable();
         }
-        return builder;
     }
 
 }

resilience-tests/src/test/java/resilience/Endpoint.java

@@ -2,6 +2,8 @@
 
 import eu.rekawek.toxiproxy.Proxy;
 
+import java.io.IOException;
+
 /**
  * class representing a proxied db endpoint
  */
@@ -42,4 +44,22 @@ public Proxy getProxy() {
     public void setProxy(Proxy proxy) {
         this.proxy = proxy;
     }
+
+    public void enable() {
+        try {
+            getProxy().enable();
+            Thread.sleep(100);
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void disable() {
+        try {
+            getProxy().disable();
+            Thread.sleep(100);
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

resilience-tests/src/test/java/resilience/SingleServerTest.java

@@ -36,7 +36,7 @@ static void afterAll() throws IOException {
 
     @BeforeEach
     void beforeEach() {
-        enableEndpoint();
+        getEndpoint().enable();
     }
 
     protected static Endpoint getEndpoint() {
@@ -49,22 +49,4 @@ protected static ArangoDB.Builder dbBuilder() {
                 .password(PASSWORD);
     }
 
-    protected void enableEndpoint(){
-        try {
-            getEndpoint().getProxy().enable();
-            Thread.sleep(100);
-        } catch (IOException | InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    protected void disableEndpoint(){
-        try {
-            getEndpoint().getProxy().disable();
-            Thread.sleep(100);
-        } catch (IOException | InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
 }

resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java

@@ -0,0 +1,219 @@
+package resilience.connection;
+
+import ch.qos.logback.classic.Level;
+import com.arangodb.*;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import resilience.ClusterTest;
+
+import java.net.ConnectException;
+import java.net.UnknownHostException;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+/**
+ * @author Michele Rastelli
+ */
+class ConnectionClusterTest extends ClusterTest {
+
+    static Stream<Protocol> protocolProvider() {
+        return Stream.of(
+                Protocol.VST,
+                Protocol.HTTP_VPACK,
+                Protocol.HTTP2_VPACK
+        );
+    }
+
+    static Stream<ArangoDB> arangoProvider() {
+        return Stream.of(
+                dbBuilder().protocol(Protocol.VST).build(),
+                dbBuilder().protocol(Protocol.HTTP_VPACK).build(),
+                dbBuilder().protocol(Protocol.HTTP2_JSON).build()
+        );
+    }
+
+    static Stream<ArangoDBAsync> asyncArangoProvider() {
+        return arangoProvider().map(ArangoDB::async);
+    }
+
+    @ParameterizedTest
+    @MethodSource("protocolProvider")
+    void nameResolutionFail(Protocol protocol) {
+        ArangoDB arangoDB = new ArangoDB.Builder()
+                .host("wrongHost", 8529)
+                .protocol(protocol)
+                .build();
+
+        Throwable thrown = catchThrowable(arangoDB::getVersion);
+        assertThat(thrown).isInstanceOf(ArangoDBException.class);
+        assertThat(thrown.getMessage()).contains("Cannot contact any host!");
+        assertThat(thrown.getCause()).isNotNull();
+        assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
+        ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> {
+            assertThat(e).isInstanceOf(UnknownHostException.class);
+            assertThat(e.getMessage()).contains("wrongHost");
+        });
+        arangoDB.shutdown();
+    }
+
+    @ParameterizedTest
+    @MethodSource("protocolProvider")
+    void nameResolutionFailAsync(Protocol protocol) {
+        ArangoDBAsync arangoDB = new ArangoDB.Builder()
+                .host("wrongHost", 8529)
+                .protocol(protocol)
+                .build()
+                .async();
+
+        Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause();
+        assertThat(thrown).isInstanceOf(ArangoDBException.class);
+        assertThat(thrown.getMessage()).contains("Cannot contact any host!");
+        assertThat(thrown.getCause()).isNotNull();
+        assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
+        ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> {
+            assertThat(e).isInstanceOf(UnknownHostException.class);
+            assertThat(e.getMessage()).contains("wrongHost");
+        });
+        arangoDB.shutdown();
+    }
+
+    @ParameterizedTest
+    @MethodSource("protocolProvider")
+    void nameResolutionFailover(Protocol protocol) {
+        ArangoDB arangoDB = new ArangoDB.Builder()
+                .password("test")
+                .host("wrongHost", 8529)
+                .host("127.0.0.1", 8529)
+                .protocol(protocol)
+                .build();
+
+        arangoDB.getVersion();
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+    }
+
+    @ParameterizedTest
+    @MethodSource("protocolProvider")
+    void nameResolutionFailoverAsync(Protocol protocol) throws ExecutionException, InterruptedException {
+        ArangoDBAsync arangoDB = new ArangoDB.Builder()
+                .password("test")
+                .host("wrongHost", 8529)
+                .host("127.0.0.1", 8529)
+                .protocol(protocol)
+                .build()
+                .async();
+
+        arangoDB.getVersion().get();
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("arangoProvider")
+    void connectionFail(ArangoDB arangoDB) {
+        disableAllEndpoints();
+
+        Throwable thrown = catchThrowable(arangoDB::getVersion);
+        assertThat(thrown).isInstanceOf(ArangoDBException.class);
+        assertThat(thrown.getMessage()).contains("Cannot contact any host");
+        assertThat(thrown.getCause()).isNotNull();
+        assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
+        ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e ->
+                assertThat(e).isInstanceOf(ConnectException.class));
+
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("asyncArangoProvider")
+    void connectionFailAsync(ArangoDBAsync arangoDB) {
+        disableAllEndpoints();
+
+        Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause();
+        assertThat(thrown).isInstanceOf(ArangoDBException.class);
+        assertThat(thrown.getMessage()).contains("Cannot contact any host");
+        assertThat(thrown.getCause()).isNotNull();
+        assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
+        ((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e ->
+                assertThat(e).isInstanceOf(ConnectException.class));
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("arangoProvider")
+    void connectionFailover(ArangoDB arangoDB) {
+        getEndpoints().get(0).disable();
+        getEndpoints().get(1).disable();
+
+        arangoDB.getVersion();
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("asyncArangoProvider")
+    void connectionFailoverAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException {
+        getEndpoints().get(0).disable();
+        getEndpoints().get(1).disable();
+
+        arangoDB.getVersion().get();
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("arangoProvider")
+    void connectionFailoverPost(ArangoDB arangoDB) {
+        getEndpoints().get(0).disable();
+        getEndpoints().get(1).disable();
+
+        arangoDB.db().query("RETURN 1", Integer.class);
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+    @ParameterizedTest(name = "{index}")
+    @MethodSource("asyncArangoProvider")
+    void connectionFailoverPostAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException {
+        getEndpoints().get(0).disable();
+        getEndpoints().get(1).disable();
+
+        arangoDB.db().query("RETURN 1", Integer.class).get();
+
+        assertThat(logs.getLogs())
+                .filteredOn(e -> e.getLevel().equals(Level.WARN))
+                .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
+
+        arangoDB.shutdown();
+        enableAllEndpoints();
+    }
+
+}