Skip to content

Commit 9003f08

Browse files
durrandurran
authored
Merge branch 'main' into NODE-5197-server-monitoring-mode
2 parents b8b5173 + 54adc9f commit 9003f08

22 files changed

+1628
-187
lines changed

‎.evergreen/ci_matrix_constants.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ const DEFAULT_OS = 'rhel80-large';
1616
const WINDOWS_OS = 'windows-vsCurrent-large';
1717
const MACOS_OS = 'macos-1100';
1818
const UBUNTU_OS = 'ubuntu1804-large';
19-
const UBUNTU_20_OS = 'ubuntu2004-small'
19+
const UBUNTU_20_OS = 'ubuntu2004-small';
20+
const UBUNTU_22_OS = 'ubuntu2204-large';
2021
const DEBIAN_OS = 'debian11-small';
2122

2223
module.exports = {
@@ -33,5 +34,6 @@ module.exports = {
3334
MACOS_OS,
3435
UBUNTU_OS,
3536
UBUNTU_20_OS,
37+
UBUNTU_22_OS,
3638
DEBIAN_OS
3739
};

‎.evergreen/config.in.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,15 @@ functions:
7676
params:
7777
script: |
7878
${PREPARE_SHELL}
79-
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
79+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
8080
- command: shell.exec
8181
params:
8282
background: true
8383
script: |
8484
${PREPARE_SHELL}
85-
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
85+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
86+
sleep 1
87+
docker ps
8688
8789
"bootstrap kms servers":
8890
- command: subprocess.exec
@@ -1090,6 +1092,13 @@ functions:
10901092
- ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
10911093

10921094
tasks:
1095+
- name: 'test-atlas-data-lake'
1096+
tags: ["datalake", "mongohouse"]
1097+
commands:
1098+
- func: 'install dependencies'
1099+
- func: 'bootstrap mongohoused'
1100+
- func: 'run data lake tests'
1101+
10931102
- name: "test-serverless"
10941103
tags: ["serverless"]
10951104
commands:

‎.evergreen/config.yml

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,15 @@ functions:
5151
params:
5252
script: |
5353
${PREPARE_SHELL}
54-
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
54+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
5555
- command: shell.exec
5656
params:
5757
background: true
5858
script: |
5959
${PREPARE_SHELL}
60-
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
60+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
61+
sleep 1
62+
docker ps
6163
bootstrap kms servers:
6264
- command: subprocess.exec
6365
params:
@@ -1033,6 +1035,14 @@ functions:
10331035
args:
10341036
- ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
10351037
tasks:
1038+
- name: test-atlas-data-lake
1039+
tags:
1040+
- datalake
1041+
- mongohouse
1042+
commands:
1043+
- func: install dependencies
1044+
- func: bootstrap mongohoused
1045+
- func: run data lake tests
10361046
- name: test-serverless
10371047
tags:
10381048
- serverless
@@ -1700,11 +1710,6 @@ tasks:
17001710
commands:
17011711
- func: install dependencies
17021712
- func: run atlas tests
1703-
- name: test-atlas-data-lake
1704-
commands:
1705-
- func: install dependencies
1706-
- func: bootstrap mongohoused
1707-
- func: run data lake tests
17081713
- name: test-5.0-load-balanced
17091714
tags:
17101715
- latest
@@ -4044,7 +4049,6 @@ buildvariants:
40444049
- test-3.6-sharded_cluster
40454050
- test-latest-server-v1-api
40464051
- test-atlas-connectivity
4047-
- test-atlas-data-lake
40484052
- test-5.0-load-balanced
40494053
- test-6.0-load-balanced
40504054
- test-latest-load-balanced
@@ -4096,7 +4100,6 @@ buildvariants:
40964100
- test-3.6-sharded_cluster
40974101
- test-latest-server-v1-api
40984102
- test-atlas-connectivity
4099-
- test-atlas-data-lake
41004103
- test-5.0-load-balanced
41014104
- test-6.0-load-balanced
41024105
- test-latest-load-balanced
@@ -4148,7 +4151,6 @@ buildvariants:
41484151
- test-3.6-sharded_cluster
41494152
- test-latest-server-v1-api
41504153
- test-atlas-connectivity
4151-
- test-atlas-data-lake
41524154
- test-5.0-load-balanced
41534155
- test-6.0-load-balanced
41544156
- test-latest-load-balanced
@@ -4199,7 +4201,6 @@ buildvariants:
41994201
- test-3.6-sharded_cluster
42004202
- test-latest-server-v1-api
42014203
- test-atlas-connectivity
4202-
- test-atlas-data-lake
42034204
- test-5.0-load-balanced
42044205
- test-6.0-load-balanced
42054206
- test-latest-load-balanced
@@ -4249,7 +4250,6 @@ buildvariants:
42494250
- test-3.6-replica_set
42504251
- test-3.6-sharded_cluster
42514252
- test-latest-server-v1-api
4252-
- test-atlas-data-lake
42534253
- test-socks5
42544254
- test-socks5-tls
42554255
- test-tls-support-latest
@@ -4292,7 +4292,6 @@ buildvariants:
42924292
- test-3.6-replica_set
42934293
- test-3.6-sharded_cluster
42944294
- test-latest-server-v1-api
4295-
- test-atlas-data-lake
42964295
- test-socks5
42974296
- test-socks5-tls
42984297
- test-tls-support-latest
@@ -4335,7 +4334,6 @@ buildvariants:
43354334
- test-3.6-replica_set
43364335
- test-3.6-sharded_cluster
43374336
- test-latest-server-v1-api
4338-
- test-atlas-data-lake
43394337
- test-socks5
43404338
- test-socks5-tls
43414339
- test-tls-support-latest
@@ -4477,6 +4475,13 @@ buildvariants:
44774475
aws-4.4-auth-test-run-aws-auth-test-AssumeRoleWithWebIdentity-with-AWS_ROLE_SESSION_NAME-unset-no-peer-dependencies
44784476
- >-
44794477
aws-4.4-auth-test-run-aws-auth-test-AssumeRoleWithWebIdentity-with-AWS_ROLE_SESSION_NAME-set-no-peer-dependencies
4478+
- name: ubuntu2204-test-atlas-data-lake
4479+
display_name: Atlas Data Lake Tests
4480+
run_on: ubuntu2204-large
4481+
expansions:
4482+
NODE_LTS_VERSION: 20
4483+
tasks:
4484+
- test-atlas-data-lake
44804485
- name: rhel8-custom-dependency-tests
44814486
display_name: Custom Dependency Version Test
44824487
run_on: rhel80-large

‎.evergreen/generate_evergreen_tasks.js

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ const {
1616
MACOS_OS,
1717
UBUNTU_OS,
1818
UBUNTU_20_OS,
19-
DEBIAN_OS
19+
DEBIAN_OS,
20+
UBUNTU_22_OS
2021
} = require('./ci_matrix_constants');
2122

2223
const OPERATING_SYSTEMS = [
@@ -120,14 +121,6 @@ TASKS.push(
120121
tags: ['atlas-connect'],
121122
commands: [{ func: 'install dependencies' }, { func: 'run atlas tests' }]
122123
},
123-
{
124-
name: 'test-atlas-data-lake',
125-
commands: [
126-
{ func: 'install dependencies' },
127-
{ func: 'bootstrap mongohoused' },
128-
{ func: 'run data lake tests' }
129-
]
130-
},
131124
{
132125
name: 'test-5.0-load-balanced',
133126
tags: ['latest', 'sharded_cluster', 'load_balancer'],
@@ -599,6 +592,16 @@ BUILD_VARIANTS.push({
599592
tasks: AWS_AUTH_TASKS
600593
});
601594

595+
BUILD_VARIANTS.push({
596+
name: 'ubuntu2204-test-atlas-data-lake',
597+
display_name: 'Atlas Data Lake Tests',
598+
run_on: UBUNTU_22_OS,
599+
expansions: {
600+
NODE_LTS_VERSION: LATEST_LTS
601+
},
602+
tasks: ['test-atlas-data-lake']
603+
});
604+
602605
const oneOffFuncAsTasks = [];
603606

604607
const FLE_PINNED_COMMIT = '974a4614f8c1c3786e5e39fa63568d83f4f69ebd';

‎src/cmap/command_monitoring_events.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import {
77
LEGACY_HELLO_COMMAND_CAMEL_CASE
88
} from '../constants';
99
import { calculateDurationInMs, deepCopy } from '../utils';
10-
import { Msg, type Query, type WriteProtocolMessageType } from './commands';
10+
import { OpMsgRequest, type OpQueryRequest, type WriteProtocolMessageType } from './commands';
1111
import type { Connection } from './connection';
1212

1313
/**
@@ -181,8 +181,8 @@ const HELLO_COMMANDS = new Set(['hello', LEGACY_HELLO_COMMAND, LEGACY_HELLO_COMM
181181

182182
// helper methods
183183
const extractCommandName = (commandDoc: Document) => Object.keys(commandDoc)[0];
184-
const namespace = (command: Query) => command.ns;
185-
const collectionName = (command: Query) => command.ns.split('.')[1];
184+
const namespace = (command: OpQueryRequest) => command.ns;
185+
const collectionName = (command: OpQueryRequest) => command.ns.split('.')[1];
186186
const maybeRedact = (commandName: string, commandDoc: Document, result: Error | Document) =>
187187
SENSITIVE_COMMANDS.has(commandName) ||
188188
(HELLO_COMMANDS.has(commandName) && commandDoc.speculativeAuthenticate)
@@ -220,7 +220,7 @@ const OP_QUERY_KEYS = [
220220

221221
/** Extract the actual command from the query, possibly up-converting if it's a legacy format */
222222
function extractCommand(command: WriteProtocolMessageType): Document {
223-
if (command instanceof Msg) {
223+
if (command instanceof OpMsgRequest) {
224224
return deepCopy(command.command);
225225
}
226226

@@ -283,7 +283,7 @@ function extractReply(command: WriteProtocolMessageType, reply?: Document) {
283283
return reply;
284284
}
285285

286-
if (command instanceof Msg) {
286+
if (command instanceof OpMsgRequest) {
287287
return deepCopy(reply.result ? reply.result : reply);
288288
}
289289

‎src/cmap/commands.ts

Lines changed: 65 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,13 @@ import { MongoInvalidArgumentError, MongoRuntimeError } from '../error';
44
import { ReadPreference } from '../read_preference';
55
import type { ClientSession } from '../sessions';
66
import type { CommandOptions } from './connection';
7-
import { OP_MSG, OP_QUERY } from './wire_protocol/constants';
7+
import {
8+
compress,
9+
Compressor,
10+
type CompressorName,
11+
uncompressibleCommands
12+
} from './wire_protocol/compression';
13+
import { OP_COMPRESSED, OP_MSG, OP_QUERY } from './wire_protocol/constants';
814

915
// Incrementing request id
1016
let _requestId = 0;
@@ -25,7 +31,7 @@ const SHARD_CONFIG_STALE = 4;
2531
const AWAIT_CAPABLE = 8;
2632

2733
/** @internal */
28-
export type WriteProtocolMessageType = Query | Msg;
34+
export type WriteProtocolMessageType = OpQueryRequest | OpMsgRequest;
2935

3036
/** @internal */
3137
export interface OpQueryOptions extends CommandOptions {
@@ -52,7 +58,7 @@ export interface OpQueryOptions extends CommandOptions {
5258
* QUERY
5359
**************************************************************/
5460
/** @internal */
55-
export class Query {
61+
export class OpQueryRequest {
5662
ns: string;
5763
numberToSkip: number;
5864
numberToReturn: number;
@@ -96,7 +102,7 @@ export class Query {
96102
this.numberToSkip = options.numberToSkip || 0;
97103
this.numberToReturn = options.numberToReturn || 0;
98104
this.returnFieldSelector = options.returnFieldSelector || undefined;
99-
this.requestId = Query.getRequestId();
105+
this.requestId = options.requestId ?? OpQueryRequest.getRequestId();
100106

101107
// special case for pre-3.2 find commands, delete ASAP
102108
this.pre32Limit = options.pre32Limit;
@@ -285,7 +291,7 @@ export interface OpResponseOptions extends BSONSerializeOptions {
285291
}
286292

287293
/** @internal */
288-
export class Response {
294+
export class OpQueryResponse {
289295
parsed: boolean;
290296
raw: Buffer;
291297
data: Buffer;
@@ -472,7 +478,7 @@ export interface OpMsgOptions {
472478
}
473479

474480
/** @internal */
475-
export class Msg {
481+
export class OpMsgRequest {
476482
requestId: number;
477483
serializeFunctions: boolean;
478484
ignoreUndefined: boolean;
@@ -502,7 +508,7 @@ export class Msg {
502508
this.options = options ?? {};
503509

504510
// Additional options
505-
this.requestId = options.requestId ? options.requestId : Msg.getRequestId();
511+
this.requestId = options.requestId ? options.requestId : OpMsgRequest.getRequestId();
506512

507513
// Serialization option
508514
this.serializeFunctions =
@@ -580,7 +586,7 @@ export class Msg {
580586
}
581587

582588
/** @internal */
583-
export class BinMsg {
589+
export class OpMsgResponse {
584590
parsed: boolean;
585591
raw: Buffer;
586592
data: Buffer;
@@ -709,3 +715,54 @@ export class BinMsg {
709715
return { utf8: { writeErrors: false } };
710716
}
711717
}
718+
719+
const MESSAGE_HEADER_SIZE = 16;
720+
const COMPRESSION_DETAILS_SIZE = 9; // originalOpcode + uncompressedSize, compressorID
721+
722+
/**
723+
* @internal
724+
*
725+
* An OP_COMPRESSED request wraps either an OP_QUERY or OP_MSG message.
726+
*/
727+
export class OpCompressedRequest {
728+
constructor(
729+
private command: WriteProtocolMessageType,
730+
private options: { zlibCompressionLevel: number; agreedCompressor: CompressorName }
731+
) {}
732+
733+
// Return whether a command contains an uncompressible command term
734+
// Will return true if command contains no uncompressible command terms
735+
static canCompress(command: WriteProtocolMessageType) {
736+
const commandDoc = command instanceof OpMsgRequest ? command.command : command.query;
737+
const commandName = Object.keys(commandDoc)[0];
738+
return !uncompressibleCommands.has(commandName);
739+
}
740+
741+
async toBin(): Promise<Buffer[]> {
742+
const concatenatedOriginalCommandBuffer = Buffer.concat(this.command.toBin());
743+
// otherwise, compress the message
744+
const messageToBeCompressed = concatenatedOriginalCommandBuffer.slice(MESSAGE_HEADER_SIZE);
745+
746+
// Extract information needed for OP_COMPRESSED from the uncompressed message
747+
const originalCommandOpCode = concatenatedOriginalCommandBuffer.readInt32LE(12);
748+
749+
// Compress the message body
750+
const compressedMessage = await compress(this.options, messageToBeCompressed);
751+
// Create the msgHeader of OP_COMPRESSED
752+
const msgHeader = Buffer.alloc(MESSAGE_HEADER_SIZE);
753+
msgHeader.writeInt32LE(
754+
MESSAGE_HEADER_SIZE + COMPRESSION_DETAILS_SIZE + compressedMessage.length,
755+
0
756+
); // messageLength
757+
msgHeader.writeInt32LE(this.command.requestId, 4); // requestID
758+
msgHeader.writeInt32LE(0, 8); // responseTo (zero)
759+
msgHeader.writeInt32LE(OP_COMPRESSED, 12); // opCode
760+
761+
// Create the compression details of OP_COMPRESSED
762+
const compressionDetails = Buffer.alloc(COMPRESSION_DETAILS_SIZE);
763+
compressionDetails.writeInt32LE(originalCommandOpCode, 0); // originalOpcode
764+
compressionDetails.writeInt32LE(messageToBeCompressed.length, 4); // Size of the uncompressed compressedMessage, excluding the MsgHeader
765+
compressionDetails.writeUInt8(Compressor[this.options.agreedCompressor], 8); // compressorID
766+
return [msgHeader, compressionDetails, compressedMessage];
767+
}
768+
}

0 commit comments

Comments
 (0)