Skip to content

Commit 2c53d6a

Browse files
rashtaorashtao
authored
[DE-722] cluster resilience tests (#526)
* cluster resilience tests * RetriableCursorClusterTest * connection failover for POST requests
1 parent 5a60478 commit 2c53d6a

13 files changed

+986
-63
lines changed

‎.github/workflows/resilience.yml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,19 @@ name: Resilience Tests
33
on:
44
workflow_dispatch:
55
push:
6-
tags: [ v** ]
6+
branches:
7+
- main
8+
- devel
9+
paths-ignore:
10+
- 'docker/**'
11+
- 'tutorial/**'
12+
- 'ChangeLog.md'
13+
- 'README.md'
14+
pull_request:
15+
types: [ opened, synchronize, reopened ]
16+
branches:
17+
- main
18+
719

820
jobs:
921
test:
@@ -26,6 +38,8 @@ jobs:
2638
cache: maven
2739
- name: Start Database
2840
run: ./docker/start_db.sh
41+
env:
42+
STARTER_MODE: cluster
2943
- name: Info
3044
run: mvn -version
3145
- name: Start Toxiproxy

‎resilience-tests/src/test/java/resilience/ClusterTest.java

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,26 +28,26 @@ public abstract class ClusterTest {
2828
@BeforeAll
2929
static void beforeAll() throws IOException {
3030
ToxiproxyClient client = new ToxiproxyClient(HOST, 8474);
31-
for (Endpoint ph : endpoints) {
32-
Proxy p = client.getProxyOrNull(ph.getName());
31+
for (Endpoint endpoint : endpoints) {
32+
Proxy p = client.getProxyOrNull(endpoint.getName());
3333
if (p != null) {
3434
p.delete();
3535
}
36-
ph.setProxy(client.createProxy(ph.getName(), ph.getHost() + ":" + ph.getPort(), ph.getUpstream()));
36+
endpoint.setProxy(client.createProxy(endpoint.getName(), endpoint.getHost() + ":" + endpoint.getPort(), endpoint.getUpstream()));
3737
}
3838
}
3939

4040
@AfterAll
4141
static void afterAll() throws IOException {
42-
for (Endpoint ph : endpoints) {
43-
ph.getProxy().delete();
42+
for (Endpoint endpoint : endpoints) {
43+
endpoint.getProxy().delete();
4444
}
4545
}
4646

4747
@BeforeEach
4848
void beforeEach() throws IOException {
49-
for (Endpoint ph : endpoints) {
50-
ph.getProxy().enable();
49+
for (Endpoint endpoint : endpoints) {
50+
endpoint.getProxy().enable();
5151
}
5252
}
5353

@@ -56,11 +56,23 @@ protected static List<Endpoint> getEndpoints() {
5656
}
5757

5858
protected static ArangoDB.Builder dbBuilder() {
59-
ArangoDB.Builder builder = new ArangoDB.Builder().password(PASSWORD);
60-
for (Endpoint ph : endpoints) {
61-
builder.host(ph.getHost(), ph.getPort());
59+
ArangoDB.Builder builder = new ArangoDB.Builder();
60+
for (Endpoint endpoint : endpoints) {
61+
builder.host(endpoint.getHost(), endpoint.getPort());
62+
}
63+
return builder.password(PASSWORD);
64+
}
65+
66+
protected void enableAllEndpoints(){
67+
for (Endpoint endpoint : endpoints) {
68+
endpoint.enable();
69+
}
70+
}
71+
72+
protected void disableAllEndpoints(){
73+
for (Endpoint endpoint : endpoints) {
74+
endpoint.disable();
6275
}
63-
return builder;
6476
}
6577

6678
}

‎resilience-tests/src/test/java/resilience/Endpoint.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import eu.rekawek.toxiproxy.Proxy;
44

5+
import java.io.IOException;
6+
57
/**
68
* class representing a proxied db endpoint
79
*/
@@ -42,4 +44,22 @@ public Proxy getProxy() {
4244
public void setProxy(Proxy proxy) {
4345
this.proxy = proxy;
4446
}
47+
48+
public void enable() {
49+
try {
50+
getProxy().enable();
51+
Thread.sleep(100);
52+
} catch (IOException | InterruptedException e) {
53+
throw new RuntimeException(e);
54+
}
55+
}
56+
57+
public void disable() {
58+
try {
59+
getProxy().disable();
60+
Thread.sleep(100);
61+
} catch (IOException | InterruptedException e) {
62+
throw new RuntimeException(e);
63+
}
64+
}
4565
}

‎resilience-tests/src/test/java/resilience/SingleServerTest.java

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ static void afterAll() throws IOException {
3636

3737
@BeforeEach
3838
void beforeEach() {
39-
enableEndpoint();
39+
getEndpoint().enable();
4040
}
4141

4242
protected static Endpoint getEndpoint() {
@@ -49,22 +49,4 @@ protected static ArangoDB.Builder dbBuilder() {
4949
.password(PASSWORD);
5050
}
5151

52-
protected void enableEndpoint(){
53-
try {
54-
getEndpoint().getProxy().enable();
55-
Thread.sleep(100);
56-
} catch (IOException | InterruptedException e) {
57-
throw new RuntimeException(e);
58-
}
59-
}
60-
61-
protected void disableEndpoint(){
62-
try {
63-
getEndpoint().getProxy().disable();
64-
Thread.sleep(100);
65-
} catch (IOException | InterruptedException e) {
66-
throw new RuntimeException(e);
67-
}
68-
}
69-
7052
}

‎resilience-tests/src/test/java/resilience/connection/ConnectionClusterTest.java

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
package resilience.connection;
2+
3+
import ch.qos.logback.classic.Level;
4+
import com.arangodb.*;
5+
import org.junit.jupiter.params.ParameterizedTest;
6+
import org.junit.jupiter.params.provider.MethodSource;
7+
import resilience.ClusterTest;
8+
9+
import java.net.ConnectException;
10+
import java.net.UnknownHostException;
11+
import java.util.concurrent.ExecutionException;
12+
import java.util.stream.Stream;
13+
14+
import static org.assertj.core.api.Assertions.assertThat;
15+
import static org.assertj.core.api.Assertions.catchThrowable;
16+
17+
/**
18+
* @author Michele Rastelli
19+
*/
20+
class ConnectionClusterTest extends ClusterTest {
21+
22+
static Stream<Protocol> protocolProvider() {
23+
return Stream.of(
24+
Protocol.VST,
25+
Protocol.HTTP_VPACK,
26+
Protocol.HTTP2_VPACK
27+
);
28+
}
29+
30+
static Stream<ArangoDB> arangoProvider() {
31+
return Stream.of(
32+
dbBuilder().protocol(Protocol.VST).build(),
33+
dbBuilder().protocol(Protocol.HTTP_VPACK).build(),
34+
dbBuilder().protocol(Protocol.HTTP2_JSON).build()
35+
);
36+
}
37+
38+
static Stream<ArangoDBAsync> asyncArangoProvider() {
39+
return arangoProvider().map(ArangoDB::async);
40+
}
41+
42+
@ParameterizedTest
43+
@MethodSource("protocolProvider")
44+
void nameResolutionFail(Protocol protocol) {
45+
ArangoDB arangoDB = new ArangoDB.Builder()
46+
.host("wrongHost", 8529)
47+
.protocol(protocol)
48+
.build();
49+
50+
Throwable thrown = catchThrowable(arangoDB::getVersion);
51+
assertThat(thrown).isInstanceOf(ArangoDBException.class);
52+
assertThat(thrown.getMessage()).contains("Cannot contact any host!");
53+
assertThat(thrown.getCause()).isNotNull();
54+
assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
55+
((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> {
56+
assertThat(e).isInstanceOf(UnknownHostException.class);
57+
assertThat(e.getMessage()).contains("wrongHost");
58+
});
59+
arangoDB.shutdown();
60+
}
61+
62+
@ParameterizedTest
63+
@MethodSource("protocolProvider")
64+
void nameResolutionFailAsync(Protocol protocol) {
65+
ArangoDBAsync arangoDB = new ArangoDB.Builder()
66+
.host("wrongHost", 8529)
67+
.protocol(protocol)
68+
.build()
69+
.async();
70+
71+
Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause();
72+
assertThat(thrown).isInstanceOf(ArangoDBException.class);
73+
assertThat(thrown.getMessage()).contains("Cannot contact any host!");
74+
assertThat(thrown.getCause()).isNotNull();
75+
assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
76+
((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e -> {
77+
assertThat(e).isInstanceOf(UnknownHostException.class);
78+
assertThat(e.getMessage()).contains("wrongHost");
79+
});
80+
arangoDB.shutdown();
81+
}
82+
83+
@ParameterizedTest
84+
@MethodSource("protocolProvider")
85+
void nameResolutionFailover(Protocol protocol) {
86+
ArangoDB arangoDB = new ArangoDB.Builder()
87+
.password("test")
88+
.host("wrongHost", 8529)
89+
.host("127.0.0.1", 8529)
90+
.protocol(protocol)
91+
.build();
92+
93+
arangoDB.getVersion();
94+
95+
assertThat(logs.getLogs())
96+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
97+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
98+
99+
arangoDB.shutdown();
100+
}
101+
102+
@ParameterizedTest
103+
@MethodSource("protocolProvider")
104+
void nameResolutionFailoverAsync(Protocol protocol) throws ExecutionException, InterruptedException {
105+
ArangoDBAsync arangoDB = new ArangoDB.Builder()
106+
.password("test")
107+
.host("wrongHost", 8529)
108+
.host("127.0.0.1", 8529)
109+
.protocol(protocol)
110+
.build()
111+
.async();
112+
113+
arangoDB.getVersion().get();
114+
115+
assertThat(logs.getLogs())
116+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
117+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
118+
119+
arangoDB.shutdown();
120+
}
121+
122+
@ParameterizedTest(name = "{index}")
123+
@MethodSource("arangoProvider")
124+
void connectionFail(ArangoDB arangoDB) {
125+
disableAllEndpoints();
126+
127+
Throwable thrown = catchThrowable(arangoDB::getVersion);
128+
assertThat(thrown).isInstanceOf(ArangoDBException.class);
129+
assertThat(thrown.getMessage()).contains("Cannot contact any host");
130+
assertThat(thrown.getCause()).isNotNull();
131+
assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
132+
((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e ->
133+
assertThat(e).isInstanceOf(ConnectException.class));
134+
135+
arangoDB.shutdown();
136+
enableAllEndpoints();
137+
}
138+
139+
@ParameterizedTest(name = "{index}")
140+
@MethodSource("asyncArangoProvider")
141+
void connectionFailAsync(ArangoDBAsync arangoDB) {
142+
disableAllEndpoints();
143+
144+
Throwable thrown = catchThrowable(() -> arangoDB.getVersion().get()).getCause();
145+
assertThat(thrown).isInstanceOf(ArangoDBException.class);
146+
assertThat(thrown.getMessage()).contains("Cannot contact any host");
147+
assertThat(thrown.getCause()).isNotNull();
148+
assertThat(thrown.getCause()).isInstanceOf(ArangoDBMultipleException.class);
149+
((ArangoDBMultipleException) thrown.getCause()).getExceptions().forEach(e ->
150+
assertThat(e).isInstanceOf(ConnectException.class));
151+
arangoDB.shutdown();
152+
enableAllEndpoints();
153+
}
154+
155+
@ParameterizedTest(name = "{index}")
156+
@MethodSource("arangoProvider")
157+
void connectionFailover(ArangoDB arangoDB) {
158+
getEndpoints().get(0).disable();
159+
getEndpoints().get(1).disable();
160+
161+
arangoDB.getVersion();
162+
163+
assertThat(logs.getLogs())
164+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
165+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
166+
167+
arangoDB.shutdown();
168+
enableAllEndpoints();
169+
}
170+
171+
@ParameterizedTest(name = "{index}")
172+
@MethodSource("asyncArangoProvider")
173+
void connectionFailoverAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException {
174+
getEndpoints().get(0).disable();
175+
getEndpoints().get(1).disable();
176+
177+
arangoDB.getVersion().get();
178+
179+
assertThat(logs.getLogs())
180+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
181+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
182+
183+
arangoDB.shutdown();
184+
enableAllEndpoints();
185+
}
186+
187+
@ParameterizedTest(name = "{index}")
188+
@MethodSource("arangoProvider")
189+
void connectionFailoverPost(ArangoDB arangoDB) {
190+
getEndpoints().get(0).disable();
191+
getEndpoints().get(1).disable();
192+
193+
arangoDB.db().query("RETURN 1", Integer.class);
194+
195+
assertThat(logs.getLogs())
196+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
197+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
198+
199+
arangoDB.shutdown();
200+
enableAllEndpoints();
201+
}
202+
203+
@ParameterizedTest(name = "{index}")
204+
@MethodSource("asyncArangoProvider")
205+
void connectionFailoverPostAsync(ArangoDBAsync arangoDB) throws ExecutionException, InterruptedException {
206+
getEndpoints().get(0).disable();
207+
getEndpoints().get(1).disable();
208+
209+
arangoDB.db().query("RETURN 1", Integer.class).get();
210+
211+
assertThat(logs.getLogs())
212+
.filteredOn(e -> e.getLevel().equals(Level.WARN))
213+
.anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host"));
214+
215+
arangoDB.shutdown();
216+
enableAllEndpoints();
217+
}
218+
219+
}

0 commit comments

Comments
 (0)