Merge branch 'main' into NODE-2422/validate_that_mongocryptd_not_spawned_if_bypassAutoEncrypt_is_true

Author: baileympearson

.evergreen/run-kms-servers.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 cd ${DRIVERS_TOOLS}/.evergreen/csfle
 . ./activate-kmstlsvenv.sh
 # by default it always runs on port 5698

.evergreen/run-typescript.sh

@@ -22,7 +22,7 @@ fi
 
 function get_ts_version() {
     if [ "$TS_VERSION" == "current" ]; then
-        echo $(node -e "console.log(require('./package-lock.json').dependencies.typescript.version)")
+        echo $(node -e "console.log(require('./package-lock.json').packages['node_modules/typescript'].version)")
     else
         echo $TS_VERSION
     fi
@@ -31,7 +31,8 @@ function get_ts_version() {
 export TSC="./node_modules/typescript/bin/tsc"
 export TS_VERSION=$(get_ts_version)
 
-npm install --no-save --force typescript@"$TS_VERSION"
+# On old versions of TS we need to put the node types back to 18.11.19
+npm install --no-save --force typescript@"$TS_VERSION" "$(if [[ $TS_VERSION == '4.1.6' ]]; then echo "@types/node@18.11.19"; else echo ""; fi)"
 
 echo "Typescript $($TSC -v)"
 

package-lock.json

@@ -51,27 +51,27 @@
   "devDependencies": {
     "@iarna/toml": "^2.2.5",
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
-    "@microsoft/api-extractor": "^7.34.4",
+    "@microsoft/api-extractor": "^7.34.8",
     "@microsoft/tsdoc-config": "^0.16.2",
     "@mongodb-js/zstd": "^1.1.0",
-    "@types/chai": "^4.3.4",
+    "@types/chai": "^4.3.5",
     "@types/chai-subset": "^1.3.3",
     "@types/express": "^4.17.17",
-    "@types/kerberos": "^1.1.1",
+    "@types/kerberos": "^1.1.2",
     "@types/mocha": "^10.0.1",
-    "@types/node": "^18.11.19",
+    "@types/node": "^20.1.0",
     "@types/saslprep": "^1.0.1",
-    "@types/semver": "^7.3.13",
-    "@types/sinon": "^10.0.13",
+    "@types/semver": "^7.5.0",
+    "@types/sinon": "^10.0.14",
     "@types/sinon-chai": "^3.2.9",
     "@types/whatwg-url": "^11.0.0",
-    "@typescript-eslint/eslint-plugin": "^5.55.0",
-    "@typescript-eslint/parser": "^5.55.0",
+    "@typescript-eslint/eslint-plugin": "^5.59.5",
+    "@typescript-eslint/parser": "^5.59.5",
     "chai": "^4.3.7",
     "chai-subset": "^1.6.0",
     "chalk": "^4.1.2",
-    "eslint": "^8.36.0",
-    "eslint-config-prettier": "^8.7.0",
+    "eslint": "^8.40.0",
+    "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-simple-import-sort": "^10.0.0",
@@ -82,9 +82,9 @@
     "mocha-sinon": "^2.1.2",
     "mongodb-legacy": "^5.0.0",
     "nyc": "^15.1.0",
-    "prettier": "^2.8.4",
-    "semver": "^7.3.8",
-    "sinon": "^15.0.2",
+    "prettier": "^2.8.8",
+    "semver": "^7.5.0",
+    "sinon": "^15.0.4",
     "sinon-chai": "^3.7.0",
     "snappy": "^7.2.2",
     "source-map-support": "^0.5.21",
@@ -94,8 +94,7 @@
     "typescript": "^4.9.5",
     "typescript-cached-transpile": "^0.0.6",
     "v8-heapsnapshot": "^1.2.0",
-    "xml2js": "^0.4.23",
-    "yargs": "^17.7.1"
+    "yargs": "^17.7.2"
   },
   "license": "Apache-2.0",
   "engines": {

package.json

@@ -251,8 +251,11 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
     this[kMetrics] = new ConnectionPoolMetrics();
     this[kProcessingWaitQueue] = false;
 
+    this.mongoLogger = this[kServer].topology.client.mongoLogger;
+    this.component = 'connection';
+
     process.nextTick(() => {
-      this.emit(ConnectionPool.CONNECTION_POOL_CREATED, new ConnectionPoolCreatedEvent(this));
+      this.emitAndLog(ConnectionPool.CONNECTION_POOL_CREATED, new ConnectionPoolCreatedEvent(this));
     });
   }
 
@@ -337,7 +340,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
       return;
     }
     this[kPoolState] = PoolState.ready;
-    this.emit(ConnectionPool.CONNECTION_POOL_READY, new ConnectionPoolReadyEvent(this));
+    this.emitAndLog(ConnectionPool.CONNECTION_POOL_READY, new ConnectionPoolReadyEvent(this));
     clearTimeout(this[kMinPoolSizeTimer]);
     this.ensureMinPoolSize();
   }
@@ -348,7 +351,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
    * explicitly destroyed by the new owner.
    */
   checkOut(callback: Callback<Connection>): void {
-    this.emit(
+    this.emitAndLog(
       ConnectionPool.CONNECTION_CHECK_OUT_STARTED,
       new ConnectionCheckOutStartedEvent(this)
     );
@@ -360,7 +363,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
         waitQueueMember[kCancelled] = true;
         waitQueueMember.timer = undefined;
 
-        this.emit(
+        this.emitAndLog(
           ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
           new ConnectionCheckOutFailedEvent(this, 'timeout')
         );
@@ -398,7 +401,10 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
     }
 
     this[kCheckedOut].delete(connection);
-    this.emit(ConnectionPool.CONNECTION_CHECKED_IN, new ConnectionCheckedInEvent(this, connection));
+    this.emitAndLog(
+      ConnectionPool.CONNECTION_CHECKED_IN,
+      new ConnectionCheckedInEvent(this, connection)
+    );
 
     if (willDestroy) {
       const reason = connection.closed ? 'error' : poolClosed ? 'poolClosed' : 'stale';
@@ -437,7 +443,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
         // Increment the generation for the service id.
         this.serviceGenerations.set(sid, generation + 1);
       }
-      this.emit(
+      this.emitAndLog(
         ConnectionPool.CONNECTION_POOL_CLEARED,
         new ConnectionPoolClearedEvent(this, { serviceId })
       );
@@ -452,9 +458,11 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
 
     this.clearMinPoolSizeTimer();
     if (!alreadyPaused) {
-      this.emit(
+      this.emitAndLog(
         ConnectionPool.CONNECTION_POOL_CLEARED,
-        new ConnectionPoolClearedEvent(this, { interruptInUseConnections })
+        new ConnectionPoolClearedEvent(this, {
+          interruptInUseConnections
+        })
       );
     }
 
@@ -509,15 +517,15 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
     eachAsync<Connection>(
       this[kConnections].toArray(),
       (conn, cb) => {
-        this.emit(
+        this.emitAndLog(
           ConnectionPool.CONNECTION_CLOSED,
           new ConnectionClosedEvent(this, conn, 'poolClosed')
         );
         conn.destroy({ force: !!options.force }, cb);
       },
       err => {
         this[kConnections].clear();
-        this.emit(ConnectionPool.CONNECTION_POOL_CLOSED, new ConnectionPoolClosedEvent(this));
+        this.emitAndLog(ConnectionPool.CONNECTION_POOL_CLOSED, new ConnectionPoolClosedEvent(this));
         callback(err);
       }
     );
@@ -645,7 +653,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
     connection: Connection,
     reason: 'error' | 'idle' | 'stale' | 'poolClosed'
   ) {
-    this.emit(
+    this.emitAndLog(
       ConnectionPool.CONNECTION_CLOSED,
       new ConnectionClosedEvent(this, connection, reason)
     );
@@ -694,15 +702,15 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
 
     this[kPending]++;
     // This is our version of a "virtual" no-I/O connection as the spec requires
-    this.emit(
+    this.emitAndLog(
       ConnectionPool.CONNECTION_CREATED,
       new ConnectionCreatedEvent(this, { id: connectOptions.id })
     );
 
     connect(connectOptions, (err, connection) => {
       if (err || !connection) {
         this[kPending]--;
-        this.emit(
+        this.emitAndLog(
           ConnectionPool.CONNECTION_CLOSED,
           new ConnectionClosedEvent(
             this,
@@ -750,7 +758,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
       }
 
       connection.markAvailable();
-      this.emit(ConnectionPool.CONNECTION_READY, new ConnectionReadyEvent(this, connection));
+      this.emitAndLog(ConnectionPool.CONNECTION_READY, new ConnectionReadyEvent(this, connection));
 
       this[kPending]--;
       callback(undefined, connection);
@@ -819,7 +827,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
       if (this[kPoolState] !== PoolState.ready) {
         const reason = this.closed ? 'poolClosed' : 'connectionError';
         const error = this.closed ? new PoolClosedError(this) : new PoolClearedError(this);
-        this.emit(
+        this.emitAndLog(
           ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
           new ConnectionCheckOutFailedEvent(this, reason, error)
         );
@@ -842,7 +850,7 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
 
       if (!this.destroyConnectionIfPerished(connection)) {
         this[kCheckedOut].add(connection);
-        this.emit(
+        this.emitAndLog(
           ConnectionPool.CONNECTION_CHECKED_OUT,
           new ConnectionCheckedOutEvent(this, connection)
         );
@@ -872,14 +880,14 @@ export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
           }
         } else {
           if (err) {
-            this.emit(
+            this.emitAndLog(
               ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
               // TODO(NODE-5192): Remove this cast
               new ConnectionCheckOutFailedEvent(this, 'connectionError', err as MongoError)
             );
           } else if (connection) {
             this[kCheckedOut].add(connection);
-            this.emit(
+            this.emitAndLog(
               ConnectionPool.CONNECTION_CHECKED_OUT,
               new ConnectionCheckedOutEvent(this, connection)
             );

src/cmap/connection_pool.ts

@@ -176,7 +176,8 @@ export function getFAASEnv(): Map<string, string | Int32> | null {
     VERCEL_REGION = ''
   } = process.env;
 
-  const isAWSFaaS = AWS_EXECUTION_ENV.length > 0 || AWS_LAMBDA_RUNTIME_API.length > 0;
+  const isAWSFaaS =
+    AWS_EXECUTION_ENV.startsWith('AWS_Lambda_') || AWS_LAMBDA_RUNTIME_API.length > 0;
   const isAzureFaaS = FUNCTIONS_WORKER_RUNTIME.length > 0;
   const isGCPFaaS = K_SERVICE.length > 0 || FUNCTION_NAME.length > 0;
   const isVercelFaaS = VERCEL.length > 0;

src/cmap/handshake/client_metadata.ts

@@ -2,6 +2,8 @@ export const MIN_SUPPORTED_SERVER_VERSION = '3.6';
 export const MAX_SUPPORTED_SERVER_VERSION = '7.0';
 export const MIN_SUPPORTED_WIRE_VERSION = 6;
 export const MAX_SUPPORTED_WIRE_VERSION = 21;
+export const MIN_SUPPORTED_QE_WIRE_VERSION = 21;
+export const MIN_SUPPORTED_QE_SERVER_VERSION = '7.0';
 export const OP_REPLY = 1;
 export const OP_UPDATE = 2001;
 export const OP_INSERT = 2002;

src/cmap/wire_protocol/constants.ts

@@ -309,11 +309,11 @@ export interface AutoEncryptionOptions {
    * Other validation rules in the JSON schema will not be enforced by the driver and will result in an error.
    */
   schemaMap?: Document;
-  /** @experimental Public Technical Preview: Supply a schema for the encrypted fields in the document  */
+  /** Supply a schema for the encrypted fields in the document  */
   encryptedFieldsMap?: Document;
   /** Allows the user to bypass auto encryption, maintaining implicit decryption */
   bypassAutoEncryption?: boolean;
-  /** @experimental Public Technical Preview: Allows users to bypass query analysis */
+  /** Allows users to bypass query analysis */
   bypassQueryAnalysis?: boolean;
   options?: {
     /** An optional hook to catch logging messages from the underlying encryption engine */

src/deps.ts

@@ -325,7 +325,7 @@ export class MongoClient extends TypedEventEmitter<MongoClientEvents> {
   /** @internal */
   topology?: Topology;
   /** @internal */
-  readonly mongoLogger: MongoLogger;
+  override readonly mongoLogger: MongoLogger;
   /** @internal */
   private connectionLock?: Promise<this>;
 
@@ -471,23 +471,21 @@ export class MongoClient extends TypedEventEmitter<MongoClientEvents> {
       }
     }
 
-    const topology = new Topology(options.hosts, options);
+    this.topology = new Topology(this, options.hosts, options);
     // Events can be emitted before initialization is complete so we have to
     // save the reference to the topology on the client ASAP if the event handlers need to access it
-    this.topology = topology;
-    topology.client = this;
 
-    topology.once(Topology.OPEN, () => this.emit('open', this));
+    this.topology.once(Topology.OPEN, () => this.emit('open', this));
 
     for (const event of MONGO_CLIENT_EVENTS) {
-      topology.on(event, (...args: any[]) => this.emit(event, ...(args as any)));
+      this.topology.on(event, (...args: any[]) => this.emit(event, ...(args as any)));
     }
 
     const topologyConnect = async () => {
       try {
-        await promisify(callback => topology.connect(options, callback))();
+        await promisify(callback => this.topology?.connect(options, callback))();
       } catch (error) {
-        topology.close({ force: true });
+        this.topology?.close({ force: true });
         throw error;
       }
     };

src/mongo_client.ts

@@ -12,6 +12,7 @@ import type {
   ObjectId,
   Timestamp
 } from './bson';
+import type { MongoLoggableComponent, MongoLogger } from './mongo_logger';
 import type { Sort } from './sort';
 
 /** @internal */
@@ -397,8 +398,21 @@ export declare interface TypedEventEmitter<Events extends EventsDescription> ext
  * Typescript type safe event emitter
  * @public
  */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-export class TypedEventEmitter<Events extends EventsDescription> extends EventEmitter {}
+
+export class TypedEventEmitter<Events extends EventsDescription> extends EventEmitter {
+  /** @internal */
+  protected mongoLogger?: MongoLogger;
+  /** @internal */
+  protected component?: MongoLoggableComponent;
+  /** @internal */
+  protected emitAndLog<EventKey extends keyof Events>(
+    event: EventKey | symbol,
+    ...args: Parameters<Events[EventKey]>
+  ): void {
+    this.emit(event, ...args);
+    if (this.component) this.mongoLogger?.debug(this.component, args[0]);
+  }
+}
 
 /** @public */
 export class CancellationToken extends TypedEventEmitter<{ cancel(): void }> {}

src/mongo_types.ts

@@ -1,6 +1,11 @@
 import type { Document } from '../bson';
+import {
+  MIN_SUPPORTED_QE_SERVER_VERSION,
+  MIN_SUPPORTED_QE_WIRE_VERSION
+} from '../cmap/wire_protocol/constants';
 import { Collection } from '../collection';
 import type { Db } from '../db';
+import { MongoCompatibilityError } from '../error';
 import type { PkFactory } from '../mongo_client';
 import type { Server } from '../sdam/server';
 import type { ClientSession } from '../sessions';
@@ -98,6 +103,10 @@ export interface CreateCollectionOptions extends CommandOperationOptions {
   changeStreamPreAndPostImages?: { enabled: boolean };
 }
 
+/* @internal */
+const INVALID_QE_VERSION =
+  'Driver support of Queryable Encryption is incompatible with server. Upgrade server to use Queryable Encryption.';
+
 /** @internal */
 export class CreateCollectionOperation extends CommandOperation<Collection> {
   override options: CreateCollectionOptions;
@@ -127,12 +136,17 @@ export class CreateCollectionOperation extends CommandOperation<Collection> {
         db.s.client.options.autoEncryption?.encryptedFieldsMap?.[`${db.databaseName}.${name}`];
 
       if (encryptedFields) {
+        // Creating a QE collection required min server of 7.0.0
+        if (server.description.maxWireVersion < MIN_SUPPORTED_QE_WIRE_VERSION) {
+          throw new MongoCompatibilityError(
+            `${INVALID_QE_VERSION} The minimum server version required is ${MIN_SUPPORTED_QE_SERVER_VERSION}`
+          );
+        }
         // Create auxilliary collections for queryable encryption support.
         const escCollection = encryptedFields.escCollection ?? `enxcol_.${name}.esc`;
-        const eccCollection = encryptedFields.eccCollection ?? `enxcol_.${name}.ecc`;
         const ecocCollection = encryptedFields.ecocCollection ?? `enxcol_.${name}.ecoc`;
 
-        for (const collectionName of [escCollection, eccCollection, ecocCollection]) {
+        for (const collectionName of [escCollection, ecocCollection]) {
           const createOp = new CreateCollectionOperation(db, collectionName, {
             clusteredIndex: {
               key: { _id: 1 },