diff --git a/.eslintignore b/.eslintignore index a2a44f452..764ef7576 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,3 @@ **/node_modules docs/dist -compiled/spectypes/build +cases/spectypes/build diff --git a/.eslintrc.json b/.eslintrc.json index 9a13a60a1..1f7f4824d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,5 +1,8 @@ { "extends": "./node_modules/gts/", + "rules": { + "node/no-unpublished-import": "off" + }, "env": { "jest": true } diff --git a/.gitignore b/.gitignore index 5ac5a0a47..d6522733b 100644 --- a/.gitignore +++ b/.gitignore @@ -80,7 +80,7 @@ typings/ # nuxt.js build output .nuxt -# react / gatsby +# react / gatsby public/ # vuepress build output @@ -99,3 +99,6 @@ public/ # sourcemaps docs/dist/app.js.map + +# spectype build artifacts +cases/spectypes/build diff --git a/README.md b/README.md index 4423011db..2b9a787be 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # ๐Ÿ“Š Benchmark Comparison of Packages with Runtime Validation and TypeScript Support +- - - - +**โšกโš  Benchmark results have changed after switching to isolated node processes for each benchmarked package, see [#864](https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864) โš โšก** +- - - - + ## Benchmark Results [![Fastest Packages - click to view details](docs/results/preview.svg)](https://moltar.github.io/typescript-runtime-type-benchmarks) @@ -74,3 +78,10 @@ function isMyDataValid(data: any) { // `res` is now type casted to the right type const res = isMyDataValid(data) ``` + +## Local Development + +* `npm run start` - run benchmarks for all modules +* `npm run start run zod myzod valita` - run benchmarks only for a few selected modules +* `npm run docs:serve` - result viewer +* `npm run test` - run tests on all modules diff --git a/benchmarks/helpers/main.ts b/benchmarks/helpers/main.ts index cdfd9354a..5e1838daf 100644 --- a/benchmarks/helpers/main.ts +++ b/benchmarks/helpers/main.ts @@ -1,5 +1,5 @@ import { add, complete, cycle, suite } from 'benny'; -import { readFileSync, writeFileSync } from 'fs'; +import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs'; import { join } from 'path'; import { writePreviewGraph } from './graph'; import { getRegisteredBenchmarks } from './register'; @@ -10,22 +10,24 @@ const NODE_VERSION = process.env.NODE_VERSION || process.version; const NODE_VERSION_FOR_PREVIEW = 17; const TEST_PREVIEW_GENERATION = false; -export async function main() { +/** + * Run all registered benchmarks and append the results to a file. + */ +export async function runAllBenchmarks() { if (TEST_PREVIEW_GENERATION) { - // just generate the preview without using benchmark data from a previous run + // during development: generate the preview using benchmark data from a previous run const allResults: BenchmarkResult[] = JSON.parse( readFileSync(join(DOCS_DIR, 'results', 'node-17.json')).toString() ).results; await writePreviewGraph({ - filename: join(DOCS_DIR, 'results', 'preview.svg'), + filename: previewSvgFilename(), values: allResults, }); return; } - const majorVersion = getNodeMajorVersion(); const allResults: BenchmarkResult[] = []; for (const [benchmark, benchmarks] of getRegisteredBenchmarks()) { @@ -42,24 +44,40 @@ export async function main() { }); } - writeFileSync( - join(DOCS_DIR, 'results', `node-${majorVersion}.json`), + // collect results of isolated benchmark runs into a single file + appendResults(allResults); +} - JSON.stringify({ - results: allResults, - }), +/** + * Remove the results json file. + */ +export function deleteResults() { + const fileName = resultsJsonFilename(); - { encoding: 'utf8' } - ); + if (existsSync(fileName)) { + unlinkSync(fileName); + } +} + +/** + * Generate the preview svg shown in the readme. + */ +export async function createPreviewGraph() { + const majorVersion = getNodeMajorVersion(); if (majorVersion === NODE_VERSION_FOR_PREVIEW) { + const allResults: BenchmarkResult[] = JSON.parse( + readFileSync(resultsJsonFilename()).toString() + ).results; + await writePreviewGraph({ - filename: join(DOCS_DIR, 'results', 'preview.svg'), + filename: previewSvgFilename(), values: allResults, }); } } +// run a benchmark fn with benny async function runBenchmarks(name: string, cases: BenchmarkCase[]) { const fns = cases.map(c => add(c.moduleName, () => c.run())); @@ -74,6 +92,52 @@ async function runBenchmarks(name: string, cases: BenchmarkCase[]) { ); } +// append results to an existing file or create a new one +function appendResults(results: BenchmarkResult[]) { + const fileName = resultsJsonFilename(); + const existingResults: BenchmarkResult[] = existsSync(fileName) + ? JSON.parse(readFileSync(fileName).toString()).results + : []; + + // check that we're appending unique data + const getKey = ({ + benchmark, + name, + nodeVersion, + }: BenchmarkResult): string => { + return JSON.stringify({ benchmark, name, nodeVersion }); + }; + const existingResultsIndex = new Set(existingResults.map(r => getKey(r))); + + results.forEach(r => { + if (existingResultsIndex.has(getKey(r))) { + console.error('Result %s already exists in', getKey(r), fileName); + + throw new Error('Duplicate result in result json file'); + } + }); + + writeFileSync( + fileName, + + JSON.stringify({ + results: [...existingResults, ...results], + }), + + { encoding: 'utf8' } + ); +} + +function resultsJsonFilename() { + const majorVersion = getNodeMajorVersion(); + + return join(DOCS_DIR, 'results', `node-${majorVersion}.json`); +} + +function previewSvgFilename() { + return join(DOCS_DIR, 'results', 'preview.svg'); +} + function getNodeMajorVersion() { let majorVersion = 0; diff --git a/benchmarks/index.ts b/benchmarks/index.ts index 48486a4a2..deb41eb40 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -1,4 +1,8 @@ -export { main } from './helpers/main'; +export { + runAllBenchmarks, + createPreviewGraph, + deleteResults, +} from './helpers/main'; export { addCase, AvailableBenchmarksIds, diff --git a/cases/index.ts b/cases/index.ts index 7d75246e7..7a93a98de 100644 --- a/cases/index.ts +++ b/cases/index.ts @@ -1,28 +1,36 @@ -import './ajv'; -import './bueno'; -import './class-validator'; -import './computed-types'; -import './decoders'; -import './io-ts'; -import './jointz'; -import './json-decoder'; -import './marshal'; -import './mojotech-json-type-validation'; -import './myzod'; -import './ok-computer'; -import './purify-ts'; -import './rulr'; -import './runtypes'; -import './simple-runtypes'; -import './spectypes'; -import './superstruct'; -import './suretype'; -import './toi'; -import './tson'; -import './ts-interface-checker'; -import './ts-json-validator'; -import './ts-utils'; -import './typeofweb-schema'; -import './valita'; -import './yup'; -import './zod'; +export const cases = [ + 'ajv', + 'bueno', + 'class-validator', + 'computed-types', + 'decoders', + 'io-ts', + 'jointz', + 'json-decoder', + 'marshal', + 'mojotech-json-type-validation', + 'myzod', + 'ok-computer', + 'purify-ts', + 'rulr', + 'runtypes', + 'simple-runtypes', + 'spectypes', + 'superstruct', + 'suretype', + 'toi', + 'ts-interface-checker', + 'ts-json-validator', + 'ts-utils', + 'tson', + 'typeofweb-schema', + 'valita', + 'yup', + 'zod', +] as const; + +export type CaseName = typeof cases[number]; + +export async function importCase(caseName: CaseName) { + await import('./' + caseName); +} diff --git a/cases/spectypes/.gitignore b/cases/spectypes/.gitignore deleted file mode 100644 index 378eac25d..000000000 --- a/cases/spectypes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build diff --git a/index.ts b/index.ts index 2b4f5fa1e..a992df613 100644 --- a/index.ts +++ b/index.ts @@ -1,4 +1,71 @@ -import { main } from './benchmarks'; -import './cases'; +import * as childProcess from 'child_process'; +import * as benchmarks from './benchmarks'; +import * as cases from './cases'; -main(); +async function main() { + // a runtype lib would be handy here to check the passed command names ;) + const [command, ...args] = process.argv.slice(2); + + switch (command) { + case undefined: + case 'run': + // run the given or all benchmarks, each in its own node process, see + // https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864 + { + console.log('Removing previous results'); + benchmarks.deleteResults(); + + const caseNames = args.length ? args : cases.cases; + + for (const c of caseNames) { + if (c === 'spectypes') { + // hack: manually run the spectypes compilation step - avoids + // having to run it before any other benchmark, esp when working + // locally and checking against a few selected ones. + childProcess.execSync('npm run compile:spectypes', { + stdio: 'inherit', + }); + } + + const cmd = [...process.argv.slice(0, 2), 'run-internal', c]; + + console.log('Executing "%s"', c); + childProcess.execSync(cmd.join(' '), { + stdio: 'inherit', + }); + } + } + break; + + case 'create-preview-svg': + // separate command, because preview generation needs the accumulated + // results from the benchmark runs + await benchmarks.createPreviewGraph(); + break; + + case 'run-internal': + // run the given benchmark(s) & append the results + { + const caseNames = args as cases.CaseName[]; + + for (const c of caseNames) { + console.log('Loading "%s"', c); + + await cases.importCase(c); + } + + await benchmarks.runAllBenchmarks(); + } + break; + + default: + console.error('unknown command:', command); + + // eslint-disable-next-line no-process-exit + process.exit(1); + } +} + +main().catch(e => { + throw e; +}); diff --git a/package.json b/package.json index 0ac6f8e6a..55df906ea 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "scripts": { "lint": "gts check", "lint:fix": "gts fix", - "start": "npm run compile:spectypes && ts-node index.ts", + "start": "ts-node index.ts", "test:build": "npm run compile:spectypes && tsc --noEmit", "test": "npm run compile:spectypes && jest", "docs:serve": "serve docs", diff --git a/start.sh b/start.sh index 8818b89fa..a51ea128f 100755 --- a/start.sh +++ b/start.sh @@ -4,4 +4,5 @@ set -ex export NODE_VERSION="${NODE_VERSION:-$(node -v)}" -npm start +npm run start +npm run start create-preview-svg diff --git a/test/benchmarks.test.ts b/test/benchmarks.test.ts index 4ca6dfbe4..29e7274a5 100644 --- a/test/benchmarks.test.ts +++ b/test/benchmarks.test.ts @@ -1,5 +1,48 @@ import { getRegisteredBenchmarks } from '../benchmarks'; -import '../cases'; +import { cases } from '../cases'; + +// all cases need to be imported here because jest cannot pic up dynamically +// imported `test` and `describe` +import '../cases/ajv'; +import '../cases/bueno'; +import '../cases/class-validator'; +import '../cases/computed-types'; +import '../cases/decoders'; +import '../cases/io-ts'; +import '../cases/jointz'; +import '../cases/json-decoder'; +import '../cases/marshal'; +import '../cases/mojotech-json-type-validation'; +import '../cases/myzod'; +import '../cases/ok-computer'; +import '../cases/purify-ts'; +import '../cases/rulr'; +import '../cases/runtypes'; +import '../cases/simple-runtypes'; +import '../cases/spectypes'; +import '../cases/superstruct'; +import '../cases/suretype'; +import '../cases/toi'; +import '../cases/ts-interface-checker'; +import '../cases/ts-json-validator'; +import '../cases/ts-utils'; +import '../cases/tson'; +import '../cases/typeofweb-schema'; +import '../cases/valita'; +import '../cases/yup'; +import '../cases/zod'; + +test('all cases must have been imported in tests', () => { + const registeredCases = new Set(); + + getRegisteredBenchmarks().forEach(nameBenchmarkPair => { + nameBenchmarkPair[1].forEach(b => { + registeredCases.add(b.moduleName); + }); + }); + + expect(registeredCases.size).toEqual(cases.length); +}); getRegisteredBenchmarks().forEach(([benchmarkId, benchmarkCases]) => { describe(benchmarkId, () => {