Skip to content

Commit 7fc1507

Browse files
katcharovkatcharov
committed
Retry flaky unified tests
JAVA-5393
1 parent 470fc2e commit 7fc1507

File tree

9 files changed

+153
-41
lines changed

9 files changed

+153
-41
lines changed

‎driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", true)
2828
}
2929
}
3030
}

‎driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", false)
2828
}
2929
}
3030
}

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c
9999
@MethodSource("data")
100100
@Override
101101
public void shouldPassAllOutcomes(
102+
final String testName,
102103
@Nullable final String fileDescription,
103104
@Nullable final String testDescription,
104105
@Nullable final String directoryName,
106+
final int attemptNumber,
107+
final int totalAttempts,
105108
final String schemaVersion,
106109
@Nullable final BsonArray runOnRequirements,
107110
final BsonArray entitiesArray,
108111
final BsonArray initialData,
109112
final BsonDocument definition) {
110113
try {
111-
super.shouldPassAllOutcomes(fileDescription,
114+
super.shouldPassAllOutcomes(
115+
testName,
116+
fileDescription,
112117
testDescription,
113118
directoryName,
119+
attemptNumber,
120+
totalAttempts,
114121
schemaVersion,
115122
runOnRequirements,
116123
entitiesArray,

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@
2424
import com.mongodb.client.unified.UnifiedTest;
2525
import com.mongodb.client.unified.UnifiedTestModifications;
2626
import com.mongodb.client.vault.ClientEncryption;
27+
import com.mongodb.lang.NonNull;
2728
import com.mongodb.reactivestreams.client.MongoClients;
2829
import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets;
2930
import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl;
3031
import com.mongodb.reactivestreams.client.syncadapter.SyncClientEncryption;
3132
import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket;
3233
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient;
3334
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase;
35+
import org.junit.jupiter.params.provider.Arguments;
36+
37+
import java.io.IOException;
38+
import java.net.URISyntaxException;
39+
import java.util.Collection;
3440

3541
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier;
3642
import static com.mongodb.client.unified.UnifiedTestModifications.TestDef;
@@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) {
94100
disableSleep();
95101
}
96102
}
103+
104+
@NonNull
105+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
106+
return getTestData(directory, true);
107+
}
97108
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525
import com.mongodb.client.gridfs.GridFSBuckets;
2626
import com.mongodb.client.internal.ClientEncryptionImpl;
2727
import com.mongodb.client.vault.ClientEncryption;
28+
import com.mongodb.lang.NonNull;
29+
import org.junit.jupiter.params.provider.Arguments;
30+
31+
import java.io.IOException;
32+
import java.net.URISyntaxException;
33+
import java.util.Collection;
2834

2935
public abstract class UnifiedSyncTest extends UnifiedTest {
3036
protected UnifiedSyncTest() {
@@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) {
4450
protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) {
4551
return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings);
4652
}
53+
54+
@NonNull
55+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
56+
return getTestData(directory, false);
57+
}
4758
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java

Lines changed: 86 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -57,14 +57,17 @@
5757
import org.junit.jupiter.params.ParameterizedTest;
5858
import org.junit.jupiter.params.provider.Arguments;
5959
import org.junit.jupiter.params.provider.MethodSource;
60+
import org.opentest4j.AssertionFailedError;
6061
import org.opentest4j.TestAbortedException;
6162

6263
import java.io.File;
6364
import java.io.IOException;
6465
import java.net.URISyntaxException;
66+
import java.text.MessageFormat;
6567
import java.util.ArrayList;
6668
import java.util.Collection;
6769
import java.util.Collections;
70+
import java.util.HashSet;
6871
import java.util.List;
6972
import java.util.Set;
7073
import java.util.concurrent.ExecutionException;
@@ -81,6 +84,7 @@
8184
import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime;
8285
import static com.mongodb.client.test.CollectionHelper.killAllSessions;
8386
import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet;
87+
import static com.mongodb.client.unified.UnifiedTestModifications.doSkips;
8488
import static com.mongodb.client.unified.UnifiedTestModifications.testDef;
8589
import static java.util.Collections.singletonList;
8690
import static java.util.stream.Collectors.toList;
@@ -101,6 +105,9 @@ public abstract class UnifiedTest {
101105
private static final Set<String> PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton(
102106
"wait queue timeout errors include details about checked out connections");
103107

108+
public static final int ATTEMPTS = 3;
109+
private static Set<String> completed = new HashSet<>();
110+
104111
@Nullable
105112
private String fileDescription;
106113
private String schemaVersion;
@@ -156,32 +163,51 @@ public Entities getEntities() {
156163
}
157164

158165
@NonNull
159-
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
166+
protected static Collection<Arguments> getTestData(final String directory, final boolean isReactive)
167+
throws URISyntaxException, IOException {
160168
List<Arguments> data = new ArrayList<>();
161169
for (File file : getTestFiles("/" + directory + "/")) {
162170
BsonDocument fileDocument = getTestDocument(file);
163-
164171
for (BsonValue cur : fileDocument.getArray("tests")) {
165-
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument()));
172+
173+
final BsonDocument testDocument = cur.asDocument();
174+
String testDescription = testDocument.getString("description").getValue();
175+
String fileDescription = fileDocument.getString("description").getValue();
176+
TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive);
177+
doSkips(testDef);
178+
179+
boolean forceFlaky = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.FORCE_FLAKY);
180+
boolean retry = forceFlaky || testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY);
181+
182+
int attempts = retry ? ATTEMPTS : 1;
183+
if (forceFlaky) {
184+
attempts = 10;
185+
}
186+
187+
for (int attempt = 1; attempt <= attempts; attempt++) {
188+
String testName = !retry
189+
? MessageFormat.format("{0}: {1}", fileDescription, testDescription)
190+
: MessageFormat.format(
191+
"{0}: {1} ({2} of {3})",
192+
fileDescription, testDescription, attempt, attempts);
193+
data.add(Arguments.of(
194+
testName,
195+
fileDescription,
196+
testDescription,
197+
directory,
198+
attempt,
199+
attempts * (forceFlaky ? -1 : 1),
200+
fileDocument.getString("schemaVersion").getValue(),
201+
fileDocument.getArray("runOnRequirements", null),
202+
fileDocument.getArray("createEntities", new BsonArray()),
203+
fileDocument.getArray("initialData", new BsonArray()),
204+
testDocument));
205+
}
166206
}
167207
}
168208
return data;
169209
}
170210

171-
@NonNull
172-
private static Arguments createTestData(
173-
final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) {
174-
return Arguments.of(
175-
fileDocument.getString("description").getValue(),
176-
testDocument.getString("description").getValue(),
177-
directory,
178-
fileDocument.getString("schemaVersion").getValue(),
179-
fileDocument.getArray("runOnRequirements", null),
180-
fileDocument.getArray("createEntities", new BsonArray()),
181-
fileDocument.getArray("initialData", new BsonArray()),
182-
testDocument);
183-
}
184-
185211
protected BsonDocument getDefinition() {
186212
return definition;
187213
}
@@ -194,9 +220,12 @@ protected BsonDocument getDefinition() {
194220

195221
@BeforeEach
196222
public void setUp(
223+
final String testName,
197224
@Nullable final String fileDescription,
198225
@Nullable final String testDescription,
199226
@Nullable final String directoryName,
227+
final int attemptNumber,
228+
final int totalAttempts,
200229
final String schemaVersion,
201230
@Nullable final BsonArray runOnRequirements,
202231
final BsonArray entitiesArray,
@@ -295,8 +324,9 @@ protected void postCleanUp(final TestDef testDef) {
295324
}
296325

297326
/**
298-
* This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)},
299-
* unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
327+
* This method is called once per
328+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless
329+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
300330
*/
301331
protected void skips(final String fileDescription, final String testDescription) {
302332
}
@@ -305,40 +335,58 @@ protected boolean isReactive() {
305335
return false;
306336
}
307337

308-
@ParameterizedTest(name = "{0}: {1}")
338+
@ParameterizedTest(name = "{0}")
309339
@MethodSource("data")
310340
public void shouldPassAllOutcomes(
341+
final String testName,
311342
@Nullable final String fileDescription,
312343
@Nullable final String testDescription,
313344
@Nullable final String directoryName,
345+
final int attemptNumber,
346+
final int totalAttempts,
314347
final String schemaVersion,
315348
@Nullable final BsonArray runOnRequirements,
316349
final BsonArray entitiesArray,
317350
final BsonArray initialData,
318351
final BsonDocument definition) {
319-
BsonArray operations = definition.getArray("operations");
320-
for (int i = 0; i < operations.size(); i++) {
321-
BsonValue cur = operations.get(i);
322-
assertOperation(rootContext, cur.asDocument(), i);
352+
boolean forceFlaky = totalAttempts < 0;
353+
if (!forceFlaky) {
354+
assumeFalse(completed.contains(testName), "Skipping retryable test that succeeded");
355+
completed.add(testName);
323356
}
357+
try {
358+
BsonArray operations = definition.getArray("operations");
359+
for (int i = 0; i < operations.size(); i++) {
360+
BsonValue cur = operations.get(i);
361+
assertOperation(rootContext, cur.asDocument(), i);
362+
}
324363

325-
if (definition.containsKey("outcome")) {
326-
assertOutcome(rootContext);
327-
}
364+
if (definition.containsKey("outcome")) {
365+
assertOutcome(rootContext);
366+
}
328367

329-
if (definition.containsKey("expectEvents")) {
330-
compareEvents(rootContext, definition);
331-
}
368+
if (definition.containsKey("expectEvents")) {
369+
compareEvents(rootContext, definition);
370+
}
332371

333-
if (definition.containsKey("expectLogMessages")) {
334-
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
335-
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
336-
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
337-
if (getMongoClientSettings().getClusterSettings()
338-
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
339-
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
372+
if (definition.containsKey("expectLogMessages")) {
373+
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
374+
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
375+
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
376+
if (getMongoClientSettings().getClusterSettings()
377+
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
378+
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
379+
}
380+
compareLogMessages(rootContext, definition, tweaks);
381+
}
382+
} catch (AssertionFailedError e) {
383+
completed.remove(testName);
384+
boolean lastAttempt = attemptNumber == Math.abs(totalAttempts);
385+
if (forceFlaky || lastAttempt) {
386+
throw e;
387+
} else {
388+
assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber);
340389
}
341-
compareLogMessages(rootContext, definition, tweaks);
342390
}
343391
}
344392

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,25 @@ final class UnifiedTestFailureValidator extends UnifiedSyncTest {
3636
@Override
3737
@BeforeEach
3838
public void setUp(
39+
final String testName,
3940
@Nullable final String fileDescription,
4041
@Nullable final String testDescription,
4142
final String directoryName,
43+
final int attemptNumber,
44+
final int totalAttempts,
4245
final String schemaVersion,
4346
@Nullable final BsonArray runOnRequirements,
4447
final BsonArray entitiesArray,
4548
final BsonArray initialData,
4649
final BsonDocument definition) {
4750
try {
4851
super.setUp(
52+
testName,
4953
fileDescription,
5054
testDescription,
5155
directoryName,
56+
attemptNumber,
57+
totalAttempts,
5258
schemaVersion,
5359
runOnRequirements,
5460
entitiesArray,
@@ -63,9 +69,12 @@ public void setUp(
6369
@ParameterizedTest
6470
@MethodSource("data")
6571
public void shouldPassAllOutcomes(
72+
final String testName,
6673
@Nullable final String fileDescription,
6774
@Nullable final String testDescription,
6875
@Nullable final String directoryName,
76+
final int attemptNumber,
77+
final int totalAttempts,
6978
final String schemaVersion,
7079
@Nullable final BsonArray runOnRequirements,
7180
final BsonArray entitiesArray,
@@ -74,9 +83,12 @@ public void shouldPassAllOutcomes(
7483
if (exception == null) {
7584
try {
7685
super.shouldPassAllOutcomes(
86+
testName,
7787
fileDescription,
7888
testDescription,
7989
directoryName,
90+
attemptNumber,
91+
totalAttempts,
8092
schemaVersion,
8193
runOnRequirements,
8294
entitiesArray,

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import static com.mongodb.ClusterFixture.isServerlessTest;
2929
import static com.mongodb.ClusterFixture.isSharded;
3030
import static com.mongodb.ClusterFixture.serverVersionLessThan;
31+
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.FORCE_FLAKY;
3132
import static com.mongodb.assertions.Assertions.assertNotNull;
3233
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS;
3334
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SKIP;
@@ -38,6 +39,14 @@
3839
public final class UnifiedTestModifications {
3940
public static void doSkips(final TestDef def) {
4041

42+
// TODO reasons for retry
43+
def.modify(FORCE_FLAKY)
44+
// Exception in encryption library: ChangeCipherSpec message sequence violation
45+
.test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider")
46+
// Number of checked out connections must match expected
47+
.test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request")
48+
.test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1");
49+
4150
// atlas-data-lake
4251

4352
def.skipAccordingToSpec("Data lake tests should only run on data lake")
@@ -478,5 +487,13 @@ public enum Modifier {
478487
* Skip the test.
479488
*/
480489
SKIP,
490+
/**
491+
* Retry the test on failure.
492+
*/
493+
RETRY,
494+
/**
495+
* Retry the test multiple times, without ignoring failures.
496+
*/
497+
FORCE_FLAKY,
481498
}
482499
}

0 commit comments

Comments
 (0)