diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f40d9880d..436d585254 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,32 @@ jobs: - name: Build package run: npm run build + fuzz: + name: Run fuzzing tests on Node v${{ matrix.node_version }} + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Setup Node.js v${{ matrix.node_version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node_version }} + + - name: Cache Node.js modules + uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.OS }}-node- + + - name: Install Dependencies + run: npm ci + + - name: Run Tests + run: npm run fuzzonly + coverage: name: Measure test coverage runs-on: ubuntu-latest @@ -114,7 +140,7 @@ jobs: github.event_name == 'push' && github.repository == 'graphql/graphql-js' && github.ref == 'refs/heads/master' - needs: [test, lint] + needs: [test, fuzz, lint] steps: - name: Checkout repo uses: actions/checkout@v2 diff --git a/.nycrc.yml b/.nycrc.yml index b057456271..ad74db4ae2 100644 --- a/.nycrc.yml +++ b/.nycrc.yml @@ -3,6 +3,7 @@ include: - 'src/' exclude: - 'src/polyfills' + - '**/*-fuzz.js' - '**/*-benchmark.js' - '**/*.d.ts' - 'src/validation/rules/ExecutableDefinitions.js' diff --git a/package.json b/package.json index bdf39204e3..7c4da5ead3 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "scripts": { "test": "npm run prettier:check && npm run lint && npm run check && npm run testonly && npm run check:ts && npm run check:spelling", "test:ci": "npm run prettier:check && npm run lint -- --no-cache && npm run check && npm run testonly:cover && npm run check:ts && npm run check:spelling && npm run build", + "fuzzonly": "mocha --full-trace src/**/__tests__/**/*-fuzz.js", "testonly": "mocha --full-trace src/**/__tests__/**/*-test.js", "testonly:cover": "nyc npm run testonly", "lint": "eslint --cache --ext .js,.ts src resources", diff --git a/src/language/__tests__/blockString-fuzz.js b/src/language/__tests__/blockString-fuzz.js new file mode 100644 index 0000000000..7f11302e6d --- /dev/null +++ b/src/language/__tests__/blockString-fuzz.js @@ -0,0 +1,68 @@ +// @flow strict + +import { describe, it } from 'mocha'; + +import dedent from '../../__testUtils__/dedent'; +import inspectStr from '../../__testUtils__/inspectStr'; +import genFuzzStrings from '../../__testUtils__/genFuzzStrings'; + +import invariant from '../../jsutils/invariant'; + +import { Lexer } from '../lexer'; +import { Source } from '../source'; +import { printBlockString } from '../blockString'; + +function lexValue(str) { + const lexer = new Lexer(new Source(str)); + const value = lexer.advance().value; + + invariant(lexer.advance().kind === '', 'Expected EOF'); + return value; +} + +describe('printBlockString', () => { + it('correctly print random strings', () => { + // Testing with length >7 is taking exponentially more time. However it is + // highly recommended to test with increased limit if you make any change. + for (const fuzzStr of genFuzzStrings({ + allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'], + maxLength: 7, + })) { + const testStr = '"""' + fuzzStr + '"""'; + + let testValue; + try { + testValue = lexValue(testStr); + } catch (e) { + continue; // skip invalid values + } + invariant(typeof testValue === 'string'); + + const printedValue = lexValue(printBlockString(testValue)); + + invariant( + testValue === printedValue, + dedent` + Expected lexValue(printBlockString(${inspectStr(testValue)})) + to equal ${inspectStr(testValue)} + but got ${inspectStr(printedValue)} + `, + ); + + const printedMultilineString = lexValue( + printBlockString(testValue, ' ', true), + ); + + invariant( + testValue === printedMultilineString, + dedent` + Expected lexValue(printBlockString(${inspectStr( + testValue, + )}, ' ', true)) + to equal ${inspectStr(testValue)} + but got ${inspectStr(printedMultilineString)} + `, + ); + } + }).timeout(20000); +}); diff --git a/src/language/__tests__/blockString-test.js b/src/language/__tests__/blockString-test.js index f4968cdafc..efd7abbd45 100644 --- a/src/language/__tests__/blockString-test.js +++ b/src/language/__tests__/blockString-test.js @@ -3,14 +3,6 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import dedent from '../../__testUtils__/dedent'; -import inspectStr from '../../__testUtils__/inspectStr'; -import genFuzzStrings from '../../__testUtils__/genFuzzStrings'; - -import invariant from '../../jsutils/invariant'; - -import { Lexer } from '../lexer'; -import { Source } from '../source'; import { dedentBlockStringValue, getBlockStringIndentation, @@ -189,57 +181,4 @@ describe('printBlockString', () => { ), ); }); - - it('correctly print random strings', () => { - // Testing with length >5 is taking exponentially more time. However it is - // highly recommended to test with increased limit if you make any change. - for (const fuzzStr of genFuzzStrings({ - allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'], - maxLength: 5, - })) { - const testStr = '"""' + fuzzStr + '"""'; - - let testValue; - try { - testValue = lexValue(testStr); - } catch (e) { - continue; // skip invalid values - } - invariant(typeof testValue === 'string'); - - const printedValue = lexValue(printBlockString(testValue)); - - invariant( - testValue === printedValue, - dedent` - Expected lexValue(printBlockString(${inspectStr(testValue)})) - to equal ${inspectStr(testValue)} - but got ${inspectStr(printedValue)} - `, - ); - - const printedMultilineString = lexValue( - printBlockString(testValue, ' ', true), - ); - - invariant( - testValue === printedMultilineString, - dedent` - Expected lexValue(printBlockString(${inspectStr( - testValue, - )}, ' ', true)) - to equal ${inspectStr(testValue)} - but got ${inspectStr(printedMultilineString)} - `, - ); - } - - function lexValue(str) { - const lexer = new Lexer(new Source(str)); - const value = lexer.advance().value; - - invariant(lexer.advance().kind === '', 'Expected EOF'); - return value; - } - }); }); diff --git a/src/utilities/__tests__/stripIgnoredCharacters-fuzz.js b/src/utilities/__tests__/stripIgnoredCharacters-fuzz.js new file mode 100644 index 0000000000..c853e1e768 --- /dev/null +++ b/src/utilities/__tests__/stripIgnoredCharacters-fuzz.js @@ -0,0 +1,53 @@ +// @flow strict + +import { describe, it } from 'mocha'; + +import dedent from '../../__testUtils__/dedent'; +import inspectStr from '../../__testUtils__/inspectStr'; +import genFuzzStrings from '../../__testUtils__/genFuzzStrings'; + +import invariant from '../../jsutils/invariant'; + +import { Lexer } from '../../language/lexer'; +import { Source } from '../../language/source'; + +import { stripIgnoredCharacters } from '../stripIgnoredCharacters'; + +function lexValue(str) { + const lexer = new Lexer(new Source(str)); + const value = lexer.advance().value; + + invariant(lexer.advance().kind === '', 'Expected EOF'); + return value; +} + +describe('stripIgnoredCharacters', () => { + it('strips ignored characters inside random block strings', () => { + // Testing with length >7 is taking exponentially more time. However it is + // highly recommended to test with increased limit if you make any change. + for (const fuzzStr of genFuzzStrings({ + allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'], + maxLength: 7, + })) { + const testStr = '"""' + fuzzStr + '"""'; + + let testValue; + try { + testValue = lexValue(testStr); + } catch (e) { + continue; // skip invalid values + } + + const strippedValue = lexValue(stripIgnoredCharacters(testStr)); + + invariant( + testValue === strippedValue, + dedent` + Expected lexValue(stripIgnoredCharacters(${inspectStr(testStr)})) + to equal ${inspectStr(testValue)} + but got ${inspectStr(strippedValue)} + `, + ); + } + }).timeout(20000); +}); diff --git a/src/utilities/__tests__/stripIgnoredCharacters-test.js b/src/utilities/__tests__/stripIgnoredCharacters-test.js index cd4e675dc9..8ac48694f0 100644 --- a/src/utilities/__tests__/stripIgnoredCharacters-test.js +++ b/src/utilities/__tests__/stripIgnoredCharacters-test.js @@ -5,7 +5,6 @@ import { describe, it } from 'mocha'; import dedent from '../../__testUtils__/dedent'; import inspectStr from '../../__testUtils__/inspectStr'; -import genFuzzStrings from '../../__testUtils__/genFuzzStrings'; import invariant from '../../jsutils/invariant'; @@ -438,35 +437,6 @@ describe('stripIgnoredCharacters', () => { expectStrippedString('"""\na\n b\nc"""').toEqual('"""a\n b\nc"""'); }); - it('strips ignored characters inside random block strings', () => { - // Testing with length >5 is taking exponentially more time. However it is - // highly recommended to test with increased limit if you make any change. - for (const fuzzStr of genFuzzStrings({ - allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'], - maxLength: 5, - })) { - const testStr = '"""' + fuzzStr + '"""'; - - let testValue; - try { - testValue = lexValue(testStr); - } catch (e) { - continue; // skip invalid values - } - - const strippedValue = lexValue(stripIgnoredCharacters(testStr)); - - invariant( - testValue === strippedValue, - dedent` - Expected lexValue(stripIgnoredCharacters(${inspectStr(testStr)})) - to equal ${inspectStr(testValue)} - but got ${inspectStr(strippedValue)} - `, - ); - } - }); - it('strips kitchen sink query but maintains the exact same AST', () => { const strippedQuery = stripIgnoredCharacters(kitchenSinkQuery); expect(stripIgnoredCharacters(strippedQuery)).to.equal(strippedQuery);