ci(NODE-6755): add tags to benchmarks (#4419)

Author: W-A-James

test/benchmarks/driver_bench/readme.md

@@ -51,13 +51,30 @@ type BenchmarkModule = {
   run: () => Promise<void>;
   afterEach?: () => Promise<void>;
   after?: () => Promise<void>;
+
+  tags?: string[];
 };
 ```
 
 Just like mocha we have once before and once after as well as before each and after each hooks.
 
 The `driver.mts` module is intended to hold various helpers for setup and teardown and help abstract some of the driver API.
 
+## Benchmark tags
+The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate
+performance alerting and filter of results via our internal tools.
+
+Tags are defined in `driver.mts` under the `TAG` enum.
+Whenever a new tag is defined it should be documented in the table below .
+
+| tag variable name |    tag string value    |                                                               purpose                                                                |
+|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+|     `TAG.spec`    | `'spec-benchmark'`     | Special tag that marks a benchmark as a spec-required benchmark                                                                      |
+|     `TAG.alert`   | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected  |
+|     `TAG.cursor`  | `'cursor-benchmark'`   | Tag marking a benchmark as being related to cursor performance                                                                       |
+|     `TAG.read`    | `'read-benchmark'`     | Tag marking a benchmark as being related to read performance                                                                         |
+|     `TAG.write`   | `'write-benchmark'`    | Tag marking a benchmark as being related to write performance                                                                        |
+
 ## Wishlist
 
 - Make it so runner can handle: `./lib/suites/multi_bench/grid_fs_upload.mjs` as an argument so shell path autocomplete makes it easier to pick a benchmark

test/benchmarks/driver_bench/src/driver.mts

@@ -7,6 +7,20 @@ import process from 'node:process';
 const __dirname = import.meta.dirname;
 const require = module.createRequire(__dirname);
 
+export const TAG = {
+  // Special tag that marks a benchmark as a spec-required benchmark
+  spec: 'spec-benchmark',
+  // Special tag that enables our perf monitoring tooling to create alerts when regressions in this
+  // benchmark's performance are detected
+  alert: 'alerting-benchmark',
+  // Tag marking a benchmark as being related to cursor performance
+  cursor: 'cursor-benchmark',
+  // Tag marking a benchmark as being related to read performance
+  read: 'read-benchmark',
+  // Tag marking a benchmark as being related to write performance
+  write: 'write-benchmark'
+};
+
 /**
  * The path to the MongoDB Node.js driver.
  * This MUST be set to the directory the driver is installed in
@@ -118,19 +132,23 @@ export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel');
 export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp');
 
 export type Metric = {
-  name: 'megabytes_per_second';
+  name: 'megabytes_per_second' | 'normalized_throughput';
   value: number;
+  metadata: {
+    improvement_direction: 'up' | 'down';
+  };
 };
 
 export type MetricInfo = {
   info: {
     test_name: string;
     args: Record<string, number>;
+    tags?: string[];
   };
   metrics: Metric[];
 };
 
-export function metrics(test_name: string, result: number): MetricInfo {
+export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo {
   return {
     info: {
       test_name,
@@ -141,9 +159,13 @@ export function metrics(test_name: string, result: number): MetricInfo {
           key,
           typeof value === 'number' ? value : value ? 1 : 0
         ])
-      )
+      ),
+      tags
     },
-    metrics: [{ name: 'megabytes_per_second', value: result }]
+    // FIXME(NODE-6781): For now all of our metrics are of throughput so their improvement_direction is up,
+    metrics: [
+      { name: 'megabytes_per_second', value: result, metadata: { improvement_direction: 'up' } }
+    ]
   } as const;
 }
 

test/benchmarks/driver_bench/src/main.mts

@@ -85,7 +85,7 @@ console.log(systemInfo());
 
 const runnerPath = path.join(__dirname, 'runner.mjs');
 
-const results: MetricInfo[] = [];
+let results: MetricInfo[] = [];
 
 for (const [suite, benchmarks] of Object.entries(tests)) {
   console.group(snakeToCamel(suite));
@@ -198,6 +198,42 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) {
   return [...results, ...compositeResults];
 }
 
-const finalResults = calculateCompositeBenchmarks(results);
+function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] {
+  const baselineBench = results.find(r => r.info.test_name === 'cpuBaseline');
+  const pingBench = results.find(r => r.info.test_name === 'ping');
+
+  assert.ok(pingBench, 'ping bench results not found!');
+  assert.ok(baselineBench, 'baseline results not found!');
+  const pingThroughput = pingBench.metrics[0].value;
+  const cpuBaseline = baselineBench.metrics[0].value;
+
+  for (const bench of results) {
+    if (bench.info.test_name === 'cpuBaseline') continue;
+    if (bench.info.test_name === 'ping') {
+      bench.metrics.push({
+        name: 'normalized_throughput',
+        value: bench.metrics[0].value / cpuBaseline,
+        metadata: {
+          improvement_direction: 'up'
+        }
+      });
+    }
+    // Compute normalized_throughput of benchmarks against ping bench
+    else {
+      bench.metrics.push({
+        name: 'normalized_throughput',
+        value: bench.metrics[0].value / pingThroughput,
+        metadata: {
+          improvement_direction: 'up'
+        }
+      });
+    }
+  }
+
+  return results;
+}
+
+results = calculateCompositeBenchmarks(results);
+results = calculateNormalizedResults(results);
 
-await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8');
+await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8');

test/benchmarks/driver_bench/src/runner.mts

@@ -14,6 +14,7 @@ type BenchmarkModule = {
   run: () => Promise<void>;
   afterEach?: () => Promise<void>;
   after?: () => Promise<void>;
+  tags?: string[];
 };
 
 const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs'));
@@ -80,6 +81,14 @@ function percentileIndex(percentile: number, count: number) {
 const medianExecution = durations[percentileIndex(50, count)];
 const megabytesPerSecond = benchmark.taskSize / medianExecution;
 
+const tags = benchmark.tags;
+if (
+  tags &&
+  (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string')))
+) {
+  throw new Error('If tags is specified, it MUST be an array of strings');
+}
+
 console.log(
   ' '.repeat(3),
   ...['total time:', totalDuration, 'sec,'],
@@ -91,6 +100,6 @@ console.log(
 
 await fs.writeFile(
   `results_${path.basename(benchmarkFile, '.mjs')}.json`,
-  JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n',
+  JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n',
   'utf8'
 );

test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts

@@ -1,8 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16.22;
 
+export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read];
+
 let collection: mongodb.Collection;
 
 export async function before() {

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts

@@ -1,10 +1,12 @@
 import { Readable, Writable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 52.43;
 
+export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read];
+
 let bucket: mongodb.GridFSBucket;
 let bin: Uint8Array;
 let _id: mongodb.ObjectId;

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts

@@ -1,9 +1,10 @@
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 52.43;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let bucket: mongodb.GridFSBucket;
 let uploadStream: mongodb.GridFSBucketWriteStream;

test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 27.31;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 2.75;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16;
+export const tags = [TAG.alert, TAG.cursor, TAG.read];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 1500;
+export const tags = [TAG.alert, TAG.cursor, TAG.read];
 
 let db: mongodb.Db;
 let tweet: Record<string, any>;

test/benchmarks/driver_bench/src/suites/node_specific/primes.mts renamed to test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts

@@ -6,12 +6,9 @@ const expectedPrimes = 78_498;
 // byteLength of
 // BSON.serialize({ primes: Buffer.from(new Int32Array(sieveOfEratosthenes(findPrimesBelow)).buffer) }).byteLength)
 // a bin data of int32s
-const byteLength = 314_010;
-
-export const taskSize = 3.1401000000000003; // ~3MB worth of work
-
-assert.equal(taskSize, byteLength * 10e-6); // taskSize should stay hardcoded, checking here the math is done right.
 
+const stableRegionMean = 42.82;
+export const taskSize = 3.1401000000000003 / stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1
 /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */
 export function sieveOfEratosthenes(n: number) {
   // Create a boolean array "prime[0..n]" and initialize

test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts

@@ -1,7 +1,9 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16.22;
 
+export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read];
+
 let collection: mongodb.Collection;
 
 export async function before() {

test/benchmarks/driver_bench/src/suites/node_specific/ping.mts

@@ -1,7 +1,8 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 // { ping: 1 } is 15 bytes of BSON x 10,000 iterations
 export const taskSize = 0.15;
+export const tags = [TAG.alert];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts

@@ -2,9 +2,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs';
 import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs';
+import { driver, type mongodb, PARALLEL_DIRECTORY, TAG, TEMP_DIRECTORY } from '../../driver.mjs';
 
 export const taskSize = 262.144;
+export const tags = [TAG.spec, TAG.alert, TAG.read, TAG.cursor];
 
 let bucket: mongodb.GridFSBucket;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts

@@ -3,9 +3,10 @@ import path from 'node:path';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
+import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs';
 
 export const taskSize = 262.144;
+export const tags = [TAG.spec, TAG.alert, TAG.write];
 
 let bucket: mongodb.GridFSBucket;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts

@@ -3,9 +3,17 @@ import path from 'node:path';
 import readline from 'node:readline/promises';
 import stream from 'node:stream/promises';
 
-import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs';
+import {
+  driver,
+  EJSON,
+  type mongodb,
+  PARALLEL_DIRECTORY,
+  TAG,
+  TEMP_DIRECTORY
+} from '../../driver.mjs';
 
 export const taskSize = 565;
+export const tags = [TAG.spec, TAG.alert, TAG.write];
 
 let collection: mongodb.Collection;
 

test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts

@@ -2,9 +2,10 @@ import { createReadStream, promises as fs } from 'node:fs';
 import path from 'node:path';
 import readline from 'node:readline/promises';
 
-import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
+import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs';
 
 export const taskSize = 565;
+export const tags = [TAG.spec, TAG.alert, TAG.write];
 
 const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi');
 let collection: mongodb.Collection;

test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16.22;
+export const tags = [TAG.spec, TAG.alert, TAG.cursor, TAG.read];
 
 let collection: mongodb.Collection<{ _id: number }>;
 

test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 27.31;
+export const tags = [TAG.spec, TAG.alert, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: Record<string, any>[];

test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts

@@ -1,7 +1,8 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 // { hello: true } is 13 bytes of BSON x 10,000 iterations
 export const taskSize = 0.13;
+export const tags = [TAG.spec, TAG.alert];
 
 let db: mongodb.Db;
 

test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 2.75;
+export const tags = [TAG.spec, TAG.alert, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: Record<string, any>[];