blockString-test: add fuzzing test for 'printBlockString' (#2574)

Author: IvanGoncharov

.babelrc.json

@@ -6,7 +6,11 @@
   ],
   "overrides": [
     {
-      "exclude": ["**/__tests__/**/*", "**/__fixtures__/**/*"],
+      "exclude": [
+        "src/__testUtils__/**/*",
+        "**/__tests__/**/*",
+        "**/__fixtures__/**/*"
+      ],
       "presets": ["@babel/preset-env"],
       "plugins": [
         ["@babel/plugin-transform-classes", { "loose": true }],

src/__testUtils__/__tests__/genFuzzStrings-test.js

@@ -0,0 +1,82 @@
+// @flow strict
+
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import genFuzzStrings from '../genFuzzStrings';
+
+function expectFuzzStrings(options) {
+  return expect(Array.from(genFuzzStrings(options)));
+}
+
+describe('genFuzzStrings', () => {
+  it('always provide empty string', () => {
+    expectFuzzStrings({ allowedChars: [], maxLength: 0 }).to.deep.equal(['']);
+    expectFuzzStrings({ allowedChars: [], maxLength: 1 }).to.deep.equal(['']);
+    expectFuzzStrings({ allowedChars: ['a'], maxLength: 0 }).to.deep.equal([
+      '',
+    ]);
+  });
+
+  it('generate strings with single character', () => {
+    expectFuzzStrings({ allowedChars: ['a'], maxLength: 1 }).to.deep.equal([
+      '',
+      'a',
+    ]);
+
+    expectFuzzStrings({
+      allowedChars: ['a', 'b', 'c'],
+      maxLength: 1,
+    }).to.deep.equal(['', 'a', 'b', 'c']);
+  });
+
+  it('generate strings with multiple character', () => {
+    expectFuzzStrings({ allowedChars: ['a'], maxLength: 2 }).to.deep.equal([
+      '',
+      'a',
+      'aa',
+    ]);
+
+    expectFuzzStrings({
+      allowedChars: ['a', 'b', 'c'],
+      maxLength: 2,
+    }).to.deep.equal([
+      '',
+      'a',
+      'b',
+      'c',
+      'aa',
+      'ab',
+      'ac',
+      'ba',
+      'bb',
+      'bc',
+      'ca',
+      'cb',
+      'cc',
+    ]);
+  });
+
+  it('generate strings longer than possible number of characters', () => {
+    expectFuzzStrings({
+      allowedChars: ['a', 'b'],
+      maxLength: 3,
+    }).to.deep.equal([
+      '',
+      'a',
+      'b',
+      'aa',
+      'ab',
+      'ba',
+      'bb',
+      'aaa',
+      'aab',
+      'aba',
+      'abb',
+      'baa',
+      'bab',
+      'bba',
+      'bbb',
+    ]);
+  });
+});

src/__testUtils__/__tests__/inspectStr-test.js

@@ -0,0 +1,21 @@
+// @flow strict
+
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import inspectStr from '../inspectStr';
+
+describe('inspectStr', () => {
+  it('handles null and undefined values', () => {
+    expect(inspectStr(null)).to.equal('null');
+    expect(inspectStr(undefined)).to.equal('null');
+  });
+
+  it('correctly print various strings', () => {
+    expect(inspectStr('')).to.equal('``');
+    expect(inspectStr('a')).to.equal('`a`');
+    expect(inspectStr('"')).to.equal('`"`');
+    expect(inspectStr("'")).to.equal("`'`");
+    expect(inspectStr('\\"')).to.equal('`\\"`');
+  });
+});

src/__testUtils__/genFuzzStrings.js

@@ -0,0 +1,31 @@
+// @flow strict
+
+/**
+ * Generator that produces all possible combinations of allowed characters.
+ */
+export default function* genFuzzStrings(options: {|
+  allowedChars: Array<string>,
+  maxLength: number,
+|}): Generator<string, void, void> {
+  const { allowedChars, maxLength } = options;
+  const numAllowedChars = allowedChars.length;
+
+  let numCombinations = 0;
+  for (let length = 1; length <= maxLength; ++length) {
+    numCombinations += numAllowedChars ** length;
+  }
+
+  yield ''; // special case for empty string
+  for (let combination = 0; combination < numCombinations; ++combination) {
+    let permutation = '';
+
+    let leftOver = combination;
+    while (leftOver >= 0) {
+      const reminder = leftOver % numAllowedChars;
+      permutation = allowedChars[reminder] + permutation;
+      leftOver = (leftOver - reminder) / numAllowedChars - 1;
+    }
+
+    yield permutation;
+  }
+}

src/__testUtils__/inspectStr.js

@@ -0,0 +1,14 @@
+// @flow strict
+
+/**
+ * Special inspect function to produce readable string literal for error messages in tests
+ */
+export default function inspectStr(str: ?string): string {
+  if (str == null) {
+    return 'null';
+  }
+  return JSON.stringify(str)
+    .replace(/^"|"$/g, '`')
+    .replace(/\\"/g, '"')
+    .replace(/\\\\/g, '\\');
+}

src/language/__tests__/blockString-test.js

@@ -3,6 +3,14 @@
 import { expect } from 'chai';
 import { describe, it } from 'mocha';
 
+import dedent from '../../__testUtils__/dedent';
+import inspectStr from '../../__testUtils__/inspectStr';
+import genFuzzStrings from '../../__testUtils__/genFuzzStrings';
+
+import invariant from '../../jsutils/invariant';
+
+import { Lexer } from '../lexer';
+import { Source } from '../source';
 import {
   dedentBlockStringValue,
   getBlockStringIndentation,
@@ -181,4 +189,57 @@ describe('printBlockString', () => {
       ),
     );
   });
+
+  it('correctly print random strings', () => {
+    // Testing with length >5 is taking exponentially more time. However it is
+    // highly recommended to test with increased limit if you make any change.
+    for (const fuzzStr of genFuzzStrings({
+      allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'],
+      maxLength: 5,
+    })) {
+      const testStr = '"""' + fuzzStr + '"""';
+
+      let testValue;
+      try {
+        testValue = lexValue(testStr);
+      } catch (e) {
+        continue; // skip invalid values
+      }
+      invariant(typeof testValue === 'string');
+
+      const printedValue = lexValue(printBlockString(testValue));
+
+      invariant(
+        testValue === printedValue,
+        dedent`
+          Expected lexValue(printBlockString(${inspectStr(testValue)}))
+            to equal ${inspectStr(testValue)}
+            but got  ${inspectStr(printedValue)}
+        `,
+      );
+
+      const printedMultilineString = lexValue(
+        printBlockString(testValue, ' ', true),
+      );
+
+      invariant(
+        testValue === printedMultilineString,
+        dedent`
+          Expected lexValue(printBlockString(${inspectStr(
+            testValue,
+          )}, ' ', true))
+            to equal ${inspectStr(testValue)}
+            but got  ${inspectStr(printedMultilineString)}
+        `,
+      );
+    }
+
+    function lexValue(str) {
+      const lexer = new Lexer(new Source(str));
+      const value = lexer.advance().value;
+
+      invariant(lexer.advance().kind === '<EOF>', 'Expected EOF');
+      return value;
+    }
+  });
 });

src/utilities/__tests__/stripIgnoredCharacters-test.js

@@ -4,6 +4,8 @@ import { expect } from 'chai';
 import { describe, it } from 'mocha';
 
 import dedent from '../../__testUtils__/dedent';
+import inspectStr from '../../__testUtils__/inspectStr';
+import genFuzzStrings from '../../__testUtils__/genFuzzStrings';
 
 import invariant from '../../jsutils/invariant';
 
@@ -67,13 +69,6 @@ function lexValue(str) {
   return value;
 }
 
-// istanbul ignore next (called only to make error messages for failing tests)
-function inspectStr(str) {
-  return (JSON.stringify(str) ?? '')
-    .replace(/^"|"$/g, '`')
-    .replace(/\\"/g, '"');
-}
-
 function expectStripped(docString) {
   return {
     toEqual(expected) {
@@ -441,45 +436,34 @@ describe('stripIgnoredCharacters', () => {
     expectStrippedString('"""\na\n b"""').toStayTheSame();
     expectStrippedString('"""\n a\n b"""').toEqual('"""a\nb"""');
     expectStrippedString('"""\na\n b\nc"""').toEqual('"""a\n b\nc"""');
+  });
 
+  it('strips ignored characters inside random block strings', () => {
     // Testing with length >5 is taking exponentially more time. However it is
     // highly recommended to test with increased limit if you make any change.
-    const maxCombinationLength = 5;
-    const possibleChars = ['\n', ' ', '"', 'a', '\\'];
-    const numPossibleChars = possibleChars.length;
-    let numCombinations = 1;
-    for (let length = 1; length < maxCombinationLength; ++length) {
-      numCombinations *= numPossibleChars;
-      for (let combination = 0; combination < numCombinations; ++combination) {
-        let testStr = '"""';
-
-        let leftOver = combination;
-        for (let i = 0; i < length; ++i) {
-          const reminder = leftOver % numPossibleChars;
-          testStr += possibleChars[reminder];
-          leftOver = (leftOver - reminder) / numPossibleChars;
-        }
-
-        testStr += '"""';
-
-        let testValue;
-        try {
-          testValue = lexValue(testStr);
-        } catch (e) {
-          continue; // skip invalid values
-        }
+    for (const fuzzStr of genFuzzStrings({
+      allowedChars: ['\n', '\t', ' ', '"', 'a', '\\'],
+      maxLength: 5,
+    })) {
+      const testStr = '"""' + fuzzStr + '"""';
+
+      let testValue;
+      try {
+        testValue = lexValue(testStr);
+      } catch (e) {
+        continue; // skip invalid values
+      }
 
-        const strippedValue = lexValue(stripIgnoredCharacters(testStr));
+      const strippedValue = lexValue(stripIgnoredCharacters(testStr));
 
-        invariant(
-          testValue === strippedValue,
-          dedent`
-            Expected lexValue(stripIgnoredCharacters(${inspectStr(testStr)}))
-              to equal ${inspectStr(testValue)}
-              but got  ${inspectStr(strippedValue)}
-          `,
-        );
-      }
+      invariant(
+        testValue === strippedValue,
+        dedent`
+          Expected lexValue(stripIgnoredCharacters(${inspectStr(testStr)}))
+            to equal ${inspectStr(testValue)}
+            but got  ${inspectStr(strippedValue)}
+        `,
+      );
     }
   });