fix(NODE-3627): Enable flexible BSON validation for server error key containing invalid utf-8

package.json

@@ -32,7 +32,7 @@
     "email": "dbx-node@mongodb.com"
   },
   "dependencies": {
-    "bson": "^4.5.4",
+    "bson": "^4.6.0",
     "denque": "^2.0.1",
     "mongodb-connection-string-url": "^2.2.0"
   },

src/bson.ts

@@ -51,6 +51,7 @@ export interface BSONSerializeOptions
       | 'cacheFunctionsCrc32'
       | 'allowObjectSmallerThanBufferSize'
       | 'index'
+      | 'validation'
     > {
   /** Return BSON filled buffers from operations */
   raw?: boolean;

src/cmap/commands.ts

@@ -469,6 +469,8 @@ export interface MessageHeader {
 export interface OpResponseOptions extends BSONSerializeOptions {
   raw?: boolean;
   documentsReturnedIn?: string | null;
+  // For now we use this internally to only prevent writeErrors from crashing the driver
+  validation?: { utf8: { writeErrors: boolean } };
 }
 
 /** @internal */
@@ -837,22 +839,24 @@ export class BinMsg {
     const promoteValues = options.promoteValues ?? this.opts.promoteValues;
     const promoteBuffers = options.promoteBuffers ?? this.opts.promoteBuffers;
     const bsonRegExp = options.bsonRegExp ?? this.opts.bsonRegExp;
+    const validation = options.validation ?? { utf8: { writeErrors: false } };
 
     // Set up the options
-    const _options: BSONSerializeOptions = {
+    const bsonOptions: BSONSerializeOptions = {
       promoteLongs,
       promoteValues,
       promoteBuffers,
-      bsonRegExp
-    };
+      bsonRegExp,
+      validation
+      // Due to the strictness of the BSON libraries validation option we need this cast
+    } as BSONSerializeOptions & { validation: { utf8: { writeErrors: boolean } } };
 
     while (this.index < this.data.length) {
       const payloadType = this.data.readUInt8(this.index++);
       if (payloadType === 0) {
         const bsonSize = this.data.readUInt32LE(this.index);
         const bin = this.data.slice(this.index, this.index + bsonSize);
-        this.documents.push(raw ? bin : BSON.deserialize(bin, _options));
-
+        this.documents.push(raw ? bin : BSON.deserialize(bin, bsonOptions));
         this.index += bsonSize;
       } else if (payloadType === 1) {
         // It was decided that no driver makes use of payload type 1
@@ -865,9 +869,8 @@ export class BinMsg {
     if (this.documents.length === 1 && documentsReturnedIn != null && raw) {
       const fieldsAsRaw: Document = {};
       fieldsAsRaw[documentsReturnedIn] = true;
-      _options.fieldsAsRaw = fieldsAsRaw;
-
-      const doc = BSON.deserialize(this.documents[0] as Buffer, _options);
+      bsonOptions.fieldsAsRaw = fieldsAsRaw;
+      const doc = BSON.deserialize(this.documents[0] as Buffer, bsonOptions);
       this.documents = [doc];
     }
 

test/unit/commands.test.ts

@@ -0,0 +1,136 @@
+import { expect } from 'chai';
+import { BinMsg, MessageHeader } from '../../src/cmap/commands';
+import { BSONError } from 'bson';
+import * as BSON from '../../src/bson';
+
+const msgHeader: MessageHeader = {
+  length: 735,
+  requestId: 14704565,
+  responseTo: 4,
+  opCode: 2013
+};
+
+// when top-level key writeErrors contains an error message that has invalid utf8
+const invalidUtf8ErrorMsg =
+  '0000000000ca020000106e00000000000477726974654572726f727300a50200000330009d02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f1000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e2982e2e2e22207d000000016f6b00000000000000f03f00';
+const msgBodyInvalidUtf8WriteErrors = Buffer.from(invalidUtf8ErrorMsg, 'hex');
+const invalidUtf8ErrorMsgDeserializeInput = Buffer.from(invalidUtf8ErrorMsg.substring(10), 'hex');
+const invalidUtf8InWriteErrorsJSON = {
+  n: 0,
+  writeErrors: [
+    {
+      index: 0,
+      code: 11000,
+      keyPattern: {
+        text: 1
+      },
+      keyValue: {
+        text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃'
+      },
+      errmsg:
+        'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }'
+    }
+  ],
+  ok: 1
+};
+
+// when another top-level key besides writeErrors has invalid utf8
+const nKeyWithInvalidUtf8 =
+  '0000000000cc020000026e0005000000f09f98ff000477726974654572726f727300a60200000330009e02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f2000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883efbfbd2e2e2e22207d000000106f6b000100000000';
+const nKeyWithInvalidUtf8DeserializeInput = Buffer.from(nKeyWithInvalidUtf8.substring(10), 'hex');
+const msgBodyNKeyWithInvalidUtf8 = Buffer.from(nKeyWithInvalidUtf8, 'hex');
+const invalidUtf8InNKeyJSON = {
+  n: '��',
+  writeErrors: [
+    {
+      index: 0,
+      code: 11000,
+      keyPattern: {
+        text: 1
+      },
+      keyValue: {
+        text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃'
+      },
+      errmsg:
+        'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }'
+    }
+  ],
+  ok: 1
+};
+
+describe('BinMsg BSON utf8 validation', () => {
+  context('when validation is disabled for writeErrors', () => {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    const options = { validation: { utf8: { writeErrors: false } as const } };
+
+    it('contains replacement characters for invalid utf8 in writeError object', () => {
+      expect(BSON.deserialize(invalidUtf8ErrorMsgDeserializeInput, options)).to.deep.equals(
+        invalidUtf8InWriteErrorsJSON
+      );
+    });
+
+    it('should not throw invalid utf8 error', () => {
+      expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw();
+    });
+  });
+
+  it('should by default disable validation for writeErrors if no validation specified', () => {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    const options = {
+      bsonRegExp: false,
+      promoteBuffers: false,
+      promoteLongs: true,
+      promoteValues: true
+    };
+    expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw();
+  });
+
+  context('when another key has invalid utf8 and validation is enabled for writeErrors', () => {
+    const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyNKeyWithInvalidUtf8
+    );
+    const options = { validation: { utf8: { writeErrors: true } as const } };
+
+    it('should not throw invalid utf8 error', () => {
+      expect(() => binMsgAnotherKeyWithInvalidUtf8.parse(options)).to.not.throw();
+    });
+
+    it('contains replacement characters for invalid utf8 key', () => {
+      expect(BSON.deserialize(nKeyWithInvalidUtf8DeserializeInput, options)).to.deep.equals(
+        invalidUtf8InNKeyJSON
+      );
+    });
+  });
+
+  it('should throw invalid utf8 error when validation enabled for writeErrors', () => {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    expect(() =>
+      binMsgInvalidUtf8ErrorMsg.parse({ validation: { utf8: { writeErrors: true } } })
+    ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document');
+  });
+
+  it('should throw error when another key has invalid utf8 and writeErrors is not validated', () => {
+    const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyNKeyWithInvalidUtf8
+    );
+    expect(() =>
+      binMsgAnotherKeyWithInvalidUtf8.parse({ validation: { utf8: { writeErrors: false } } })
+    ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document');
+  });
+});