From da5e31e20720f39ff36a2c3adc2b97925bd47875 Mon Sep 17 00:00:00 2001 From: yosuke ota Date: Thu, 3 Nov 2022 00:02:10 +0900 Subject: [PATCH 1/3] feat: add support for non-existent file parsing --- src/ts.ts | 155 ++++++++++++++++++++++++------ tests/src/types.ts | 71 +------------- tests/src/{ => utils}/fixtures.ts | 6 +- tests/src/utils/utils.ts | 67 +++++++++++++ tests/src/virtual-file.ts | 138 ++++++++++++++++++++++++++ 5 files changed, 342 insertions(+), 95 deletions(-) rename tests/src/{ => utils}/fixtures.ts (86%) create mode 100644 tests/src/utils/utils.ts create mode 100644 tests/src/virtual-file.ts diff --git a/src/ts.ts b/src/ts.ts index c398c52..b143860 100644 --- a/src/ts.ts +++ b/src/ts.ts @@ -18,7 +18,6 @@ export class TSServiceManager { public getProgram(code: string, options: ProgramOptions): ts.Program { const tsconfigPath = options.project; - const fileName = normalizeFileName(toAbsolutePath(options.filePath)); const extraFileExtensions = [...new Set(options.extraFileExtensions)]; let serviceList = this.tsServices.get(tsconfigPath); @@ -37,7 +36,7 @@ export class TSServiceManager { serviceList.unshift(service); } - return service.getProgram(code, fileName); + return service.getProgram(code, options.filePath); } } @@ -49,22 +48,59 @@ export class TSService { private currTarget = { code: "", filePath: "", + dirMap: new Map(), }; private readonly fileWatchCallbacks = new Map void>(); + private readonly dirWatchCallbacks = new Map void>(); + public constructor(tsconfigPath: string, extraFileExtensions: string[]) { this.extraFileExtensions = extraFileExtensions; this.watch = this.createWatch(tsconfigPath, extraFileExtensions); } public getProgram(code: string, filePath: string): ts.Program { - const lastTargetFilePath = this.currTarget.filePath; + const normalized = normalizeFileName( + toRealFileName(filePath, this.extraFileExtensions) + ); + const lastTarget = this.currTarget; + + const dirMap = new Map(); + let childPath = normalized; + for (const dirName of iterateDirs(normalized)) { + dirMap.set(dirName, { path: childPath, name: path.basename(childPath) }); + childPath = dirName; + } this.currTarget = { code, - filePath, + filePath: normalized, + dirMap, }; - const refreshTargetPaths = [filePath, lastTargetFilePath].filter((s) => s); + for (const { filePath: targetPath, dirMap: map } of [ + this.currTarget, + lastTarget, + ]) { + if (!targetPath) continue; + if (ts.sys.fileExists(targetPath)) { + getFileNamesIncludingVirtualTSX( + targetPath, + this.extraFileExtensions + ).forEach((vFilePath) => { + this.fileWatchCallbacks.get(vFilePath)?.(); + }); + } else { + // Signal a directory change to request a re-scan of the directory + // because it targets a file that does not actually exist. + for (const dirName of map.keys()) { + this.dirWatchCallbacks.get(dirName)?.(); + } + } + } + + const refreshTargetPaths = [normalized, lastTarget.filePath].filter( + (s) => s + ); for (const targetPath of refreshTargetPaths) { getFileNamesIncludingVirtualTSX( targetPath, @@ -84,9 +120,7 @@ export class TSService { tsconfigPath: string, extraFileExtensions: string[] ): ts.WatchOfConfigFile { - const normalizedTsconfigPaths = new Set([ - normalizeFileName(toAbsolutePath(tsconfigPath)), - ]); + const normalizedTsconfigPaths = new Set([normalizeFileName(tsconfigPath)]); const watchCompilerHost = ts.createWatchCompilerHost( tsconfigPath, { @@ -120,17 +154,41 @@ export class TSService { fileExists: watchCompilerHost.fileExists, // eslint-disable-next-line @typescript-eslint/unbound-method -- Store original readDirectory: watchCompilerHost.readDirectory, + // eslint-disable-next-line @typescript-eslint/unbound-method -- Store original + directoryExists: watchCompilerHost.directoryExists!, + // eslint-disable-next-line @typescript-eslint/unbound-method -- Store original + getDirectories: watchCompilerHost.getDirectories!, + }; + watchCompilerHost.getDirectories = (dirName, ...args) => { + return distinctArray( + ...original.getDirectories.call(watchCompilerHost, dirName, ...args), + // Include the path to the target file if the target file does not actually exist. + this.currTarget.dirMap.get(normalizeFileName(dirName))?.name + ); + }; + watchCompilerHost.directoryExists = (dirName, ...args) => { + return ( + original.directoryExists.call(watchCompilerHost, dirName, ...args) || + // Include the path to the target file if the target file does not actually exist. + this.currTarget.dirMap.has(normalizeFileName(dirName)) + ); }; - watchCompilerHost.readDirectory = (...args) => { - const results = original.readDirectory.call(watchCompilerHost, ...args); - - return [ - ...new Set( - results.map((result) => - toVirtualTSXFileName(result, extraFileExtensions) - ) - ), - ]; + watchCompilerHost.readDirectory = (dirName, ...args) => { + const results = original.readDirectory.call( + watchCompilerHost, + dirName, + ...args + ); + + // Include the target file if the target file does not actually exist. + const file = this.currTarget.dirMap.get(normalizeFileName(dirName)); + if (file && file.path === this.currTarget.filePath) { + results.push(file.path); + } + + return distinctArray(...results).map((result) => + toVirtualTSXFileName(result, extraFileExtensions) + ); }; watchCompilerHost.readFile = (fileName, ...args) => { const realFileName = toRealFileName(fileName, extraFileExtensions); @@ -151,12 +209,14 @@ export class TSService { if (!code) { return code; } + // If it's tsconfig, it will take care of rewriting the `include`. if (normalizedTsconfigPaths.has(normalized)) { const configJson = ts.parseConfigFileTextToJson(realFileName, code); if (!configJson.config) { return code; } if (configJson.config.extends) { + // If it references another tsconfig, rewrite the `include` for that file as well. for (const extendConfigPath of [configJson.config.extends].flat()) { normalizedTsconfigPaths.add( normalizeFileName( @@ -184,12 +244,28 @@ export class TSService { }); }; // Modify it so that it can be determined that the virtual file actually exists. - watchCompilerHost.fileExists = (fileName, ...args) => - original.fileExists.call( + watchCompilerHost.fileExists = (fileName, ...args) => { + const normalizedFileName = normalizeFileName(fileName); + + // Even if it is actually a file, if it is specified as a directory to the target file, + // it is assumed that it does not exist as a file. + if (this.currTarget.dirMap.has(normalizedFileName)) { + return false; + } + const normalizedRealFileName = toRealFileName( + normalizedFileName, + extraFileExtensions + ); + if (this.currTarget.filePath === normalizedRealFileName) { + // It is the file currently being parsed. + return true; + } + return original.fileExists.call( watchCompilerHost, toRealFileName(fileName, extraFileExtensions), ...args ); + }; // It keeps a callback to mark the parsed file as changed so that it can be reparsed. watchCompilerHost.watchFile = (fileName, callback) => { @@ -205,11 +281,15 @@ export class TSService { }; }; // Use watchCompilerHost but don't actually watch the files and directories. - watchCompilerHost.watchDirectory = () => ({ - close() { - // noop - }, - }); + watchCompilerHost.watchDirectory = (dirName, callback) => { + const normalized = normalizeFileName(dirName); + this.dirWatchCallbacks.set(normalized, () => callback(dirName)); + return { + close: () => { + this.dirWatchCallbacks.delete(normalized); + }, + }; + }; /** * It heavily references typescript-eslint. @@ -278,13 +358,32 @@ function normalizeFileName(fileName: string) { normalized = normalized.slice(0, -1); } if (ts.sys.useCaseSensitiveFileNames) { - return normalized; + return toAbsolutePath(normalized, null); } - return normalized.toLowerCase(); + return toAbsolutePath(normalized.toLowerCase(), null); } -function toAbsolutePath(filePath: string, baseDir?: string) { +function toAbsolutePath(filePath: string, baseDir: string | null) { return path.isAbsolute(filePath) ? filePath : path.join(baseDir || process.cwd(), filePath); } + +function* iterateDirs(filePath: string) { + let target = filePath; + let parent: string; + while ((parent = path.dirname(target)) !== target) { + yield parent; + target = parent; + } +} + +function distinctArray(...list: (string | null | undefined)[]) { + return [ + ...new Set( + ts.sys.useCaseSensitiveFileNames + ? list.map((s) => s?.toLowerCase()) + : list + ), + ].filter((s): s is string => s != null); +} diff --git a/tests/src/types.ts b/tests/src/types.ts index dc0797b..9b78240 100644 --- a/tests/src/types.ts +++ b/tests/src/types.ts @@ -12,17 +12,16 @@ import * as vueParser from "vue-eslint-parser"; import * as svelteParser from "svelte-eslint-parser"; import * as astroParser from "astro-eslint-parser"; import * as tsParser from "../../src"; -import type * as tsEslintParser from "@typescript-eslint/parser"; import semver from "semver"; import assert from "assert"; -import { iterateFixtures } from "./fixtures"; +import { iterateFixtures } from "./utils/fixtures"; +import { buildTypes } from "./utils/utils"; //------------------------------------------------------------------------------ // Helpers //------------------------------------------------------------------------------ const ROOT = path.join(__dirname, "../fixtures/types"); -const PROJECT_ROOT = path.join(__dirname, "../.."); const PARSER_OPTIONS = { comment: true, ecmaVersion: 2020, @@ -32,68 +31,6 @@ const PARSER_OPTIONS = { parser: tsParser, }; -function buildTypes( - input: string, - result: ReturnType -) { - const tsNodeMap = result.services.esTreeNodeToTSNodeMap; - const checker = - result.services.program && result.services.program.getTypeChecker(); - - const checked = new Set(); - - const lines = input.split(/\r?\n/); - const types: string[][] = []; - - function addType(node: any) { - const tsNode = tsNodeMap.get(node); - const type = checker.getTypeAtLocation(tsNode); - const typeText = checker.typeToString(type); - const lineTypes = - types[node.loc.start.line - 1] || (types[node.loc.start.line - 1] = []); - if (node.type === "Identifier") { - lineTypes.push(`${node.name}: ${typeText}`); - } else { - lineTypes.push(`${input.slice(...node.range)}: ${typeText}`); - } - } - - vueParser.AST.traverseNodes(result.ast as any, { - visitorKeys: result.visitorKeys as any, - enterNode(node, parent) { - if (checked.has(parent)) { - checked.add(node); - return; - } - - if ( - node.type === "CallExpression" || - node.type === "Identifier" || - node.type === "MemberExpression" - ) { - addType(node); - checked.add(node); - } - }, - leaveNode() { - // noop - }, - }); - return lines - .map((l, i) => { - if (!types[i]) { - return l; - } - return `${l} // ${types[i].join(", ").replace(/\n\s*/g, " ")}`; - }) - .join("\n") - .replace(new RegExp(escapeRegExp(PROJECT_ROOT), "gu"), ""); -} - -function escapeRegExp(string: string) { - return string.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&"); -} - //------------------------------------------------------------------------------ // Main //------------------------------------------------------------------------------ @@ -132,7 +69,7 @@ describe("Template Types", () => { continue; } - describe(`'test/fixtures/ast/${name}/${path.basename(sourcePath)}'`, () => { + describe(`'test/fixtures/${name}/${path.basename(sourcePath)}'`, () => { it("should be parsed to valid Types.", () => { const result = path.extname(sourcePath) === ".vue" @@ -141,6 +78,8 @@ describe("Template Types", () => { ? svelteParser.parseForESLint(source, options) : path.extname(sourcePath) === ".astro" ? astroParser.parseForESLint(source, options) + : path.extname(sourcePath) === ".ts" + ? tsParser.parseForESLint(source, options) : vueParser.parseForESLint(source, options); const actual = buildTypes(source, result as any); const resultPath = sourcePath.replace(/source\.([a-z]+)$/u, "types.$1"); diff --git a/tests/src/fixtures.ts b/tests/src/utils/fixtures.ts similarity index 86% rename from tests/src/fixtures.ts rename to tests/src/utils/fixtures.ts index aa860b1..2e5ab9a 100644 --- a/tests/src/fixtures.ts +++ b/tests/src/utils/fixtures.ts @@ -7,7 +7,11 @@ export function iterateFixtures(baseDir: string): Iterable<{ sourcePath: string; tsconfigPath: string; }> { - return iterateFixturesWithTsConfig(baseDir, ""); + const tsconfigPathCandidate = path.join(baseDir, `tsconfig.json`); + const tsconfigPath = fs.existsSync(tsconfigPathCandidate) + ? tsconfigPathCandidate + : ""; + return iterateFixturesWithTsConfig(baseDir, tsconfigPath); } function* iterateFixturesWithTsConfig( diff --git a/tests/src/utils/utils.ts b/tests/src/utils/utils.ts new file mode 100644 index 0000000..629fe93 --- /dev/null +++ b/tests/src/utils/utils.ts @@ -0,0 +1,67 @@ +import type * as tsEslintParser from "@typescript-eslint/parser"; +import path from "path"; +import * as vueParser from "vue-eslint-parser"; + +const PROJECT_ROOT = path.join(__dirname, "../../.."); + +export function buildTypes( + input: string, + result: ReturnType +): string { + const tsNodeMap = result.services.esTreeNodeToTSNodeMap; + const checker = + result.services.program && result.services.program.getTypeChecker(); + + const checked = new Set(); + + const lines = input.split(/\r?\n/); + const types: string[][] = []; + + function addType(node: any) { + const tsNode = tsNodeMap.get(node); + const type = checker.getTypeAtLocation(tsNode); + const typeText = checker.typeToString(type); + const lineTypes = + types[node.loc.start.line - 1] || (types[node.loc.start.line - 1] = []); + if (node.type === "Identifier") { + lineTypes.push(`${node.name}: ${typeText}`); + } else { + lineTypes.push(`${input.slice(...node.range)}: ${typeText}`); + } + } + + vueParser.AST.traverseNodes(result.ast as any, { + visitorKeys: result.visitorKeys as any, + enterNode(node, parent) { + if (checked.has(parent)) { + checked.add(node); + return; + } + + if ( + node.type === "CallExpression" || + node.type === "Identifier" || + node.type === "MemberExpression" + ) { + addType(node); + checked.add(node); + } + }, + leaveNode() { + // noop + }, + }); + return lines + .map((l, i) => { + if (!types[i]) { + return l; + } + return `${l} // ${types[i].join(", ").replace(/\n\s*/g, " ")}`; + }) + .join("\n") + .replace(new RegExp(escapeRegExp(PROJECT_ROOT), "gu"), ""); +} + +function escapeRegExp(string: string) { + return string.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&"); +} diff --git a/tests/src/virtual-file.ts b/tests/src/virtual-file.ts new file mode 100644 index 0000000..074d320 --- /dev/null +++ b/tests/src/virtual-file.ts @@ -0,0 +1,138 @@ +/** + * @author Yosuke Ota + */ + +//------------------------------------------------------------------------------ +// Requirements +//------------------------------------------------------------------------------ + +import fs from "fs"; +import path from "path"; +import * as vueParser from "vue-eslint-parser"; +import * as tsParser from "../../src"; +import semver from "semver"; +import assert from "assert"; +import { buildTypes } from "./utils/utils"; +import { iterateFixtures } from "./utils/fixtures"; + +//------------------------------------------------------------------------------ +// Helpers +//------------------------------------------------------------------------------ + +const ROOT = path.join(__dirname, "../fixtures/types/vue"); +const PARSER_OPTIONS = { + comment: true, + ecmaVersion: 2020, + loc: true, + range: true, + tokens: true, + parser: tsParser, +}; + +//------------------------------------------------------------------------------ +// Main +//------------------------------------------------------------------------------ + +describe("Virtual Files", () => { + for (const { name, sourcePath, filePath, tsconfigPath } of iterateFixtures( + ROOT + )) { + // if (!sourcePath.endsWith(".ts")) continue; + const optionsPath = path.join(filePath, `parser-options.json`); + const requirementsPath = path.join(filePath, `requirements.json`); + const source = fs.readFileSync(sourcePath, "utf8"); + const parserOptions = fs.existsSync(optionsPath) + ? JSON.parse(fs.readFileSync(optionsPath, "utf8")) + : {}; + const requirements = fs.existsSync(requirementsPath) + ? JSON.parse(fs.readFileSync(requirementsPath, "utf8")) + : {}; + const options = Object.assign( + { filePath: sourcePath }, + PARSER_OPTIONS, + { project: tsconfigPath }, + parserOptions + ); + + if ( + Object.entries(requirements).some(([pkgName, pkgVersion]) => { + const version = + pkgName === "node" + ? process.version + : // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires -- ignore + require(`${pkgName}/package.json`).version; + return !semver.satisfies(version, pkgVersion as string); + }) + ) { + continue; + } + + describe(`'test/fixtures/vue/${name}/${path.basename( + options.filePath + )}/v.ts'`, () => { + it("should be parsed to valid Types.", () => { + const virtualSource = source.replace(/from "\./gu, 'from "../.'); + const virtualOptions = { + ...options, + filePath: `${options.filePath}/v.ts`, + }; + const result = + path.extname(sourcePath) === ".ts" + ? tsParser.parseForESLint(virtualSource, virtualOptions) + : vueParser.parseForESLint(virtualSource, virtualOptions); + + const actual = buildTypes(virtualSource, result as any).replace( + /from "\.\.\/\./gu, + 'from ".' + ); + const resultPath = sourcePath.replace(/source\.([a-z]+)$/u, "types.$1"); + const expected = fs.readFileSync(resultPath, "utf8"); + assert.strictEqual(actual, expected); + }); + }); + describe(`'test/fixtures/vue/${name}/${path.basename( + options.filePath + )}/v/v.ts'`, () => { + it("should be parsed to valid Types.", () => { + const virtualSource = source.replace(/from "\./gu, 'from "../../.'); + const virtualOptions = { + ...options, + filePath: `${options.filePath}/v/v.ts`, + }; + const result = + path.extname(sourcePath) === ".ts" + ? tsParser.parseForESLint(virtualSource, virtualOptions) + : vueParser.parseForESLint(virtualSource, virtualOptions); + const actual = buildTypes(virtualSource, result as any).replace( + /from "\.\.\/\.\.\/\./gu, + 'from ".' + ); + const resultPath = sourcePath.replace(/source\.([a-z]+)$/u, "types.$1"); + const expected = fs.readFileSync(resultPath, "utf8"); + assert.strictEqual(actual, expected); + }); + }); + describe(`'test/fixtures/vue/${name}/${path.basename( + options.filePath + )}/v/v.vue'`, () => { + it("should be parsed to valid Types.", () => { + const virtualSource = source.replace(/from "\./gu, 'from "../../.'); + const virtualOptions = { + ...options, + filePath: `${options.filePath}/v/v.vue`, + }; + const result = + path.extname(sourcePath) === ".ts" + ? tsParser.parseForESLint(virtualSource, virtualOptions) + : vueParser.parseForESLint(virtualSource, virtualOptions); + const actual = buildTypes(virtualSource, result as any).replace( + /from "\.\.\/\.\.\/\./gu, + 'from ".' + ); + const resultPath = sourcePath.replace(/source\.([a-z]+)$/u, "types.$1"); + const expected = fs.readFileSync(resultPath, "utf8"); + assert.strictEqual(actual, expected); + }); + }); + } +}); From 95527ebe12fe085bebe1dbd2f633d228978fb1f7 Mon Sep 17 00:00:00 2001 From: Yosuke Ota Date: Thu, 3 Nov 2022 00:03:00 +0900 Subject: [PATCH 2/3] Create light-turkeys-mix.md --- .changeset/light-turkeys-mix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/light-turkeys-mix.md diff --git a/.changeset/light-turkeys-mix.md b/.changeset/light-turkeys-mix.md new file mode 100644 index 0000000..c34012b --- /dev/null +++ b/.changeset/light-turkeys-mix.md @@ -0,0 +1,5 @@ +--- +"typescript-eslint-parser-for-extra-files": minor +--- + +feat: add support for non-existent file parsing From 3d15185b5f971b5ab9d712d63e3820b0ebe11e8f Mon Sep 17 00:00:00 2001 From: yosuke ota Date: Thu, 3 Nov 2022 08:40:09 +0900 Subject: [PATCH 3/3] update --- src/ts.ts | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/src/ts.ts b/src/ts.ts index b143860..42aedbe 100644 --- a/src/ts.ts +++ b/src/ts.ts @@ -43,6 +43,8 @@ export class TSServiceManager { export class TSService { private readonly watch: ts.WatchOfConfigFile; + private readonly tsconfigPath: string; + public readonly extraFileExtensions: string[]; private currTarget = { @@ -53,9 +55,8 @@ export class TSService { private readonly fileWatchCallbacks = new Map void>(); - private readonly dirWatchCallbacks = new Map void>(); - public constructor(tsconfigPath: string, extraFileExtensions: string[]) { + this.tsconfigPath = tsconfigPath; this.extraFileExtensions = extraFileExtensions; this.watch = this.createWatch(tsconfigPath, extraFileExtensions); } @@ -77,31 +78,13 @@ export class TSService { filePath: normalized, dirMap, }; - for (const { filePath: targetPath, dirMap: map } of [ - this.currTarget, - lastTarget, - ]) { + for (const { filePath: targetPath } of [this.currTarget, lastTarget]) { if (!targetPath) continue; - if (ts.sys.fileExists(targetPath)) { - getFileNamesIncludingVirtualTSX( - targetPath, - this.extraFileExtensions - ).forEach((vFilePath) => { - this.fileWatchCallbacks.get(vFilePath)?.(); - }); - } else { + if (!ts.sys.fileExists(targetPath)) { // Signal a directory change to request a re-scan of the directory // because it targets a file that does not actually exist. - for (const dirName of map.keys()) { - this.dirWatchCallbacks.get(dirName)?.(); - } + this.fileWatchCallbacks.get(normalizeFileName(this.tsconfigPath))?.(); } - } - - const refreshTargetPaths = [normalized, lastTarget.filePath].filter( - (s) => s - ); - for (const targetPath of refreshTargetPaths) { getFileNamesIncludingVirtualTSX( targetPath, this.extraFileExtensions @@ -281,12 +264,10 @@ export class TSService { }; }; // Use watchCompilerHost but don't actually watch the files and directories. - watchCompilerHost.watchDirectory = (dirName, callback) => { - const normalized = normalizeFileName(dirName); - this.dirWatchCallbacks.set(normalized, () => callback(dirName)); + watchCompilerHost.watchDirectory = () => { return { close: () => { - this.dirWatchCallbacks.delete(normalized); + // noop }, }; };