Skip to content

Commit ee3073e

Browse files
committed
Remove the BSON document size validation requirement for the client bulk write operation
JAVA-5695
1 parent 6f68c0e commit ee3073e

File tree

7 files changed

+16
-121
lines changed

7 files changed

+16
-121
lines changed

driver-core/src/main/com/mongodb/internal/connection/BsonWriterHelper.java

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,9 @@ static EncodeDocumentsResult writeDocumentsOfDualMessageSequences(
108108
final int commandDocumentSizeInBytes,
109109
final BsonOutput firstOutput,
110110
final BsonOutput secondOutput,
111-
final MessageSettings messageSettings,
112-
final boolean validateDocumentSizeLimits) {
113-
BsonBinaryWriter firstWriter = createBsonBinaryWriter(
114-
firstOutput, dualMessageSequences.getFirstFieldNameValidator(), validateDocumentSizeLimits ? messageSettings : null);
115-
BsonBinaryWriter secondWriter = createBsonBinaryWriter(
116-
secondOutput, dualMessageSequences.getSecondFieldNameValidator(), validateDocumentSizeLimits ? messageSettings : null);
111+
final MessageSettings messageSettings) {
112+
BsonBinaryWriter firstWriter = createBsonBinaryWriter(firstOutput, dualMessageSequences.getFirstFieldNameValidator(), null);
113+
BsonBinaryWriter secondWriter = createBsonBinaryWriter(secondOutput, dualMessageSequences.getSecondFieldNameValidator(), null);
117114
// the size of operation-agnostic command fields (a.k.a. extra elements) is counted towards `messageOverheadInBytes`
118115
int messageOverheadInBytes = 1000;
119116
int maxSizeInBytes = messageSettings.getMaxMessageSize() - (messageOverheadInBytes + commandDocumentSizeInBytes);
@@ -231,45 +228,6 @@ private static boolean exceedsLimits(final MessageSettings settings, final int m
231228
return false;
232229
}
233230

234-
/**
235-
* @return {@code writer} if {@code maxDocumentSize} is {@code null}, otherwise decorates it.
236-
*/
237-
public static BsonWriter decorateWithDocumentSizeChecking(final BsonBinaryWriter writer, @Nullable final Integer maxDocumentSize) {
238-
return maxDocumentSize == null ? writer : new DocumentSizeLimitCheckingBsonBinaryWriter(writer, maxDocumentSize);
239-
}
240-
241231
private BsonWriterHelper() {
242232
}
243-
244-
private static final class DocumentSizeLimitCheckingBsonBinaryWriter extends LevelCountingBsonWriter {
245-
private final int maxStoredDocumentSize;
246-
private final BsonOutput out;
247-
private int documentStart;
248-
249-
DocumentSizeLimitCheckingBsonBinaryWriter(final BsonBinaryWriter writer, final int maxStoredDocumentSize) {
250-
super(writer);
251-
assertTrue(maxStoredDocumentSize > 0);
252-
this.maxStoredDocumentSize = maxStoredDocumentSize;
253-
this.out = writer.getBsonOutput();
254-
}
255-
256-
@Override
257-
public void writeStartDocument() {
258-
if (getCurrentLevel() == INITIAL_LEVEL) {
259-
documentStart = out.getPosition();
260-
}
261-
super.writeStartDocument();
262-
}
263-
264-
@Override
265-
public void writeEndDocument() throws BsonMaximumSizeExceededException {
266-
super.writeEndDocument();
267-
if (getCurrentLevel() == INITIAL_LEVEL) {
268-
int documentSize = out.getPosition() - documentStart;
269-
if (documentSize > maxStoredDocumentSize) {
270-
throw createBsonMaximumSizeExceededException(maxStoredDocumentSize);
271-
}
272-
}
273-
}
274-
}
275233
}

driver-core/src/main/com/mongodb/internal/connection/CommandMessage.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -231,10 +231,8 @@ protected EncodingMetadata encodeMessageBodyWithMetadata(final ByteBufferBsonOut
231231
bsonOutput.writeByte(0); // payload type
232232
commandStartPosition = bsonOutput.getPosition();
233233
ArrayList<BsonElement> extraElements = getExtraElements(operationContext);
234-
// `DualMessageSequences` requires validation only if no response is expected, otherwise we must rely on the server validation
235-
boolean validateDocumentSizeLimits = !(sequences instanceof DualMessageSequences) || !responseExpected;
236234

237-
int commandDocumentSizeInBytes = writeDocument(command, bsonOutput, commandFieldNameValidator, validateDocumentSizeLimits);
235+
int commandDocumentSizeInBytes = writeDocument(command, bsonOutput, commandFieldNameValidator);
238236
if (sequences instanceof SplittablePayload) {
239237
appendElementsToDocument(bsonOutput, commandStartPosition, extraElements);
240238
SplittablePayload payload = (SplittablePayload) sequences;
@@ -254,7 +252,7 @@ bsonOutput, getSettings(), messageStartPosition, payload, getSettings().getMaxDo
254252
writeOpMsgSectionWithPayloadType1(bsonOutputBranch2, dualMessageSequences.getSecondSequenceId(), () ->
255253
writeDocumentsOfDualMessageSequences(
256254
dualMessageSequences, commandDocumentSizeInBytes, bsonOutputBranch1,
257-
bsonOutputBranch2, getSettings(), validateDocumentSizeLimits)
255+
bsonOutputBranch2, getSettings())
258256
)
259257
);
260258
dualMessageSequencesRequireResponse = encodeDocumentsResult.isServerResponseRequired();
@@ -282,7 +280,7 @@ bsonOutputBranch2, getSettings(), validateDocumentSizeLimits)
282280
elements = new ArrayList<>(3);
283281
addServerApiElements(elements);
284282
}
285-
writeDocument(command, bsonOutput, commandFieldNameValidator, true);
283+
writeDocument(command, bsonOutput, commandFieldNameValidator);
286284
appendElementsToDocument(bsonOutput, commandStartPosition, elements);
287285
}
288286
return new EncodingMetadata(commandStartPosition);

driver-core/src/main/com/mongodb/internal/connection/RequestMessage.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,8 @@ protected void writeMessagePrologue(final BsonOutput bsonOutput) {
153153
*/
154154
protected abstract EncodingMetadata encodeMessageBodyWithMetadata(ByteBufferBsonOutput bsonOutput, OperationContext operationContext);
155155

156-
protected int writeDocument(final BsonDocument document, final BsonOutput bsonOutput,
157-
final FieldNameValidator validator, final boolean validateMaxBsonObjectSize) {
158-
BsonBinaryWriter writer = createBsonBinaryWriter(bsonOutput, validator, validateMaxBsonObjectSize ? getSettings() : null);
156+
protected int writeDocument(final BsonDocument document, final BsonOutput bsonOutput, final FieldNameValidator validator) {
157+
BsonBinaryWriter writer = createBsonBinaryWriter(bsonOutput, validator, getSettings());
159158
int documentStart = bsonOutput.getPosition();
160159
encodeUsingRegistry(writer, document);
161160
return bsonOutput.getPosition() - documentStart;

driver-core/src/main/com/mongodb/internal/operation/ClientBulkWriteOperation.java

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@
112112
import static com.mongodb.assertions.Assertions.fail;
113113
import static com.mongodb.internal.VisibleForTesting.AccessModifier.PACKAGE;
114114
import static com.mongodb.internal.VisibleForTesting.AccessModifier.PRIVATE;
115-
import static com.mongodb.internal.connection.BsonWriterHelper.decorateWithDocumentSizeChecking;
116115
import static com.mongodb.internal.connection.DualMessageSequences.WritersProviderAndLimitsChecker.WriteResult.FAIL_LIMIT_EXCEEDED;
117116
import static com.mongodb.internal.connection.DualMessageSequences.WritersProviderAndLimitsChecker.WriteResult.OK_LIMIT_NOT_REACHED;
118117
import static com.mongodb.internal.operation.BulkWriteBatch.logWriteModelDoesNotSupportRetries;
@@ -234,8 +233,7 @@ private Integer executeBatch(
234233
retryState.attach(AttachmentKeys.maxWireVersion(), connectionDescription.getMaxWireVersion(), true)
235234
.attach(AttachmentKeys.commandDescriptionSupplier(), () -> BULK_WRITE_COMMAND_NAME, false);
236235
ClientBulkWriteCommand bulkWriteCommand = createBulkWriteCommand(
237-
retryState, effectiveRetryWrites, effectiveWriteConcern, sessionContext, unexecutedModels,
238-
connectionDescription.getMaxDocumentSize(), batchEncoder,
236+
retryState, effectiveRetryWrites, effectiveWriteConcern, sessionContext, unexecutedModels, batchEncoder,
239237
() -> retryState.attach(AttachmentKeys.retryableCommandFlag(), true, true));
240238
return executeBulkWriteCommandAndExhaustOkResponse(
241239
retryState, connectionSource, connection, bulkWriteCommand, effectiveWriteConcern, operationContext);
@@ -342,7 +340,6 @@ private ClientBulkWriteCommand createBulkWriteCommand(
342340
final WriteConcern effectiveWriteConcern,
343341
final SessionContext sessionContext,
344342
final List<? extends ClientNamespacedWriteModel> unexecutedModels,
345-
final int maxStoredDocumentSize,
346343
final BatchEncoder batchEncoder,
347344
final Runnable retriesEnabler) {
348345
BsonDocument commandDocument = new BsonDocument(BULK_WRITE_COMMAND_NAME, new BsonInt32(1))
@@ -360,8 +357,6 @@ private ClientBulkWriteCommand createBulkWriteCommand(
360357
commandDocument,
361358
new ClientBulkWriteCommand.OpsAndNsInfo(
362359
effectiveRetryWrites, unexecutedModels,
363-
// we must validate the size only if no response is expected, otherwise we must rely on the server validation
364-
effectiveWriteConcern.isAcknowledged() ? null : maxStoredDocumentSize,
365360
batchEncoder,
366361
options,
367362
() -> {
@@ -679,8 +674,6 @@ OpsAndNsInfo getOpsAndNsInfo() {
679674
public static final class OpsAndNsInfo extends DualMessageSequences {
680675
private final boolean effectiveRetryWrites;
681676
private final List<? extends ClientNamespacedWriteModel> models;
682-
@Nullable
683-
private final Integer maxStoredDocumentSize;
684677
private final BatchEncoder batchEncoder;
685678
private final ConcreteClientBulkWriteOptions options;
686679
private final Supplier<Long> doIfCommandIsRetryableAndAdvanceGetTxnNumber;
@@ -689,14 +682,12 @@ public static final class OpsAndNsInfo extends DualMessageSequences {
689682
public OpsAndNsInfo(
690683
final boolean effectiveRetryWrites,
691684
final List<? extends ClientNamespacedWriteModel> models,
692-
@Nullable final Integer maxStoredDocumentSize,
693685
final BatchEncoder batchEncoder,
694686
final ConcreteClientBulkWriteOptions options,
695687
final Supplier<Long> doIfCommandIsRetryableAndAdvanceGetTxnNumber) {
696688
super("ops", new OpsFieldNameValidator(models), "nsInfo", NoOpFieldNameValidator.INSTANCE);
697689
this.effectiveRetryWrites = effectiveRetryWrites;
698690
this.models = models;
699-
this.maxStoredDocumentSize = maxStoredDocumentSize;
700691
this.batchEncoder = batchEncoder;
701692
this.options = options;
702693
this.doIfCommandIsRetryableAndAdvanceGetTxnNumber = doIfCommandIsRetryableAndAdvanceGetTxnNumber;
@@ -720,8 +711,7 @@ public EncodeDocumentsResult encodeDocuments(final WritersProviderAndLimitsCheck
720711
boolean writeNewNamespace = indexedNamespaces.size() != indexedNamespacesSizeBeforeCompute;
721712
int finalModelIndexInBatch = modelIndexInBatch;
722713
writeResult = writersProviderAndLimitsChecker.tryWrite((opsWriter, nsInfoWriter) -> {
723-
batchEncoder.encodeWriteModel(opsWriter, maxStoredDocumentSize,
724-
namespacedModel.getModel(), finalModelIndexInBatch, namespaceIndexInBatch);
714+
batchEncoder.encodeWriteModel(opsWriter, namespacedModel.getModel(), finalModelIndexInBatch, namespaceIndexInBatch);
725715
if (writeNewNamespace) {
726716
nsInfoWriter.writeStartDocument();
727717
nsInfoWriter.writeString("ns", namespace.getFullName());
@@ -957,7 +947,7 @@ public BatchEncoder() {
957947
/**
958948
* Must be called at most once.
959949
* Must not be called before calling
960-
* {@link #encodeWriteModel(BsonBinaryWriter, Integer, ClientWriteModel, int, int)} at least once.
950+
* {@link #encodeWriteModel(BsonBinaryWriter, ClientWriteModel, int, int)} at least once.
961951
* Renders {@code this} unusable.
962952
*/
963953
EncodedBatchInfo intoEncodedBatchInfo() {
@@ -979,15 +969,14 @@ void reset(final int modelIndexInBatch) {
979969

980970
void encodeWriteModel(
981971
final BsonBinaryWriter writer,
982-
@Nullable final Integer maxStoredDocumentSize,
983972
final ClientWriteModel model,
984973
final int modelIndexInBatch,
985974
final int namespaceIndexInBatch) {
986975
assertNotNull(encodedBatchInfo).modelsCount++;
987976
writer.writeStartDocument();
988977
if (model instanceof ConcreteClientInsertOneModel) {
989978
writer.writeInt32("insert", namespaceIndexInBatch);
990-
encodeWriteModelInternals(writer, maxStoredDocumentSize, (ConcreteClientInsertOneModel) model, modelIndexInBatch);
979+
encodeWriteModelInternals(writer, (ConcreteClientInsertOneModel) model, modelIndexInBatch);
991980
} else if (model instanceof ConcreteClientUpdateOneModel) {
992981
writer.writeInt32("update", namespaceIndexInBatch);
993982
writer.writeBoolean("multi", false);
@@ -998,7 +987,7 @@ void encodeWriteModel(
998987
encodeWriteModelInternals(writer, (ConcreteClientUpdateManyModel) model);
999988
} else if (model instanceof ConcreteClientReplaceOneModel) {
1000989
writer.writeInt32("update", namespaceIndexInBatch);
1001-
encodeWriteModelInternals(writer, maxStoredDocumentSize, (ConcreteClientReplaceOneModel) model);
990+
encodeWriteModelInternals(writer, (ConcreteClientReplaceOneModel) model);
1002991
} else if (model instanceof ConcreteClientDeleteOneModel) {
1003992
writer.writeInt32("delete", namespaceIndexInBatch);
1004993
writer.writeBoolean("multi", false);
@@ -1015,14 +1004,13 @@ void encodeWriteModel(
10151004

10161005
private void encodeWriteModelInternals(
10171006
final BsonBinaryWriter writer,
1018-
@Nullable final Integer maxStoredDocumentSize,
10191007
final ConcreteClientInsertOneModel model,
10201008
final int modelIndexInBatch) {
10211009
writer.writeName("document");
10221010
Object document = model.getDocument();
10231011
assertNotNull(encodedBatchInfo).insertModelDocumentIds.compute(modelIndexInBatch, (k, knownModelDocumentId) -> {
10241012
IdHoldingBsonWriter documentIdHoldingBsonWriter = new IdHoldingBsonWriter(
1025-
decorateWithDocumentSizeChecking(writer, maxStoredDocumentSize),
1013+
writer,
10261014
// Reuse `knownModelDocumentId` if it may have been generated by `IdHoldingBsonWriter` in a previous attempt.
10271015
// If its type is not `BsonObjectId`, we know it could not have been generated.
10281016
knownModelDocumentId instanceof BsonObjectId ? knownModelDocumentId.asObjectId() : null);
@@ -1061,16 +1049,12 @@ private void encodeWriteModelInternals(final BsonWriter writer, final AbstractCl
10611049
options.isUpsert().ifPresent(value -> writer.writeBoolean("upsert", value));
10621050
}
10631051

1064-
private void encodeWriteModelInternals(
1065-
final BsonBinaryWriter writer,
1066-
@Nullable final Integer maxStoredDocumentSize,
1067-
final ConcreteClientReplaceOneModel model) {
1052+
private void encodeWriteModelInternals(final BsonBinaryWriter writer, final ConcreteClientReplaceOneModel model) {
10681053
writer.writeBoolean("multi", false);
10691054
writer.writeName("filter");
10701055
encodeUsingRegistry(writer, model.getFilter());
10711056
writer.writeName("updateMods");
1072-
encodeUsingRegistry(decorateWithDocumentSizeChecking(writer, maxStoredDocumentSize), model.getReplacement(),
1073-
COLLECTIBLE_DOCUMENT_ENCODER_CONTEXT);
1057+
encodeUsingRegistry(writer, model.getReplacement(), COLLECTIBLE_DOCUMENT_ENCODER_CONTEXT);
10741058
ConcreteClientReplaceOptions options = model.getOptions();
10751059
options.getCollation().ifPresent(value -> {
10761060
writer.writeName("collation");

driver-core/src/test/unit/com/mongodb/internal/connection/CommandMessageTest.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,6 @@ void getCommandDocumentFromClientBulkWrite() {
141141
OpsAndNsInfo opsAndNsInfo = new OpsAndNsInfo(
142142
retryWrites,
143143
writeModels,
144-
null,
145144
new ClientBulkWriteOperation(
146145
writeModels,
147146
clientBulkWriteOptions(),

driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/CrudProseTest.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,6 @@ protected void testBulkWriteHandlesCursorRequiringGetMoreWithinTransaction() {
6060
assumeTrue(java.lang.Boolean.parseBoolean(toString()), "BULK-TODO implement");
6161
}
6262

63-
@DisplayName("10. MongoClient.bulkWrite returns error for unacknowledged too-large insert")
64-
@ParameterizedTest
65-
@MethodSource("testBulkWriteErrorsForUnacknowledgedTooLargeInsertArgs")
66-
@Override
67-
protected void testBulkWriteErrorsForUnacknowledgedTooLargeInsert(final String operationType, final boolean nesting) {
68-
assumeTrue(java.lang.Boolean.parseBoolean(toString()), "BULK-TODO implement");
69-
}
70-
7163
@DisplayName("11. MongoClient.bulkWrite batch splits when the addition of a new namespace exceeds the maximum message size")
7264
@Test
7365
@Override

driver-sync/src/test/functional/com/mongodb/client/CrudProseTest.java

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import com.mongodb.MongoNamespace;
2525
import com.mongodb.MongoWriteConcernException;
2626
import com.mongodb.MongoWriteException;
27-
import com.mongodb.WriteConcern;
2827
import com.mongodb.assertions.Assertions;
2928
import com.mongodb.client.model.CreateCollectionOptions;
3029
import com.mongodb.client.model.Filters;
@@ -313,40 +312,6 @@ private void assertBulkWriteHandlesCursorRequiringGetMore(final boolean transact
313312
}
314313
}
315314

316-
/**
317-
* This test extends the one required by the specification.
318-
*/
319-
@DisplayName("10. MongoClient.bulkWrite returns error for unacknowledged too-large insert")
320-
@ParameterizedTest
321-
@MethodSource("testBulkWriteErrorsForUnacknowledgedTooLargeInsertArgs")
322-
protected void testBulkWriteErrorsForUnacknowledgedTooLargeInsert(final String operationType, final boolean nesting) {
323-
assumeTrue(serverVersionAtLeast(8, 0));
324-
assumeFalse(isServerlessTest());
325-
try (MongoClient client = createMongoClient(getMongoClientSettingsBuilder()
326-
.writeConcern(WriteConcern.UNACKNOWLEDGED))) {
327-
int maxBsonObjectSize = droppedDatabase(client).runCommand(new Document("hello", 1)).getInteger("maxBsonObjectSize");
328-
Document document = nesting
329-
? new Document("a", join("", nCopies(maxBsonObjectSize / 2, "b")))
330-
.append("nested", new Document("c", join("", nCopies(maxBsonObjectSize / 2, "d"))))
331-
: new Document("a", join("", nCopies(maxBsonObjectSize, "b")));
332-
ClientNamespacedWriteModel model;
333-
switch (operationType) {
334-
case "insert": {
335-
model = ClientNamespacedWriteModel.insertOne(NAMESPACE, document);
336-
break;
337-
}
338-
case "replace": {
339-
model = ClientNamespacedWriteModel.replaceOne(NAMESPACE, Filters.empty(), document);
340-
break;
341-
}
342-
default: {
343-
throw Assertions.fail(operationType);
344-
}
345-
}
346-
assertThrows(BsonMaximumSizeExceededException.class, () -> client.bulkWrite(singletonList(model)));
347-
}
348-
}
349-
350315
private static Stream<Arguments> testBulkWriteErrorsForUnacknowledgedTooLargeInsertArgs() {
351316
return Stream.of(
352317
arguments("insert", false),

0 commit comments

Comments
 (0)