Switch to fast-glob instead of glob + limit files analyzed

This should ensure that the LSP server does not crash and yield
a major performance increase for large repositories.

fast-glob returns the same results and is around 10x faster.

Author: skovhus

server/src/analyser.ts

@@ -7,7 +7,7 @@ import { promisify } from 'util'
 import * as LSP from 'vscode-languageserver'
 import * as Parser from 'web-tree-sitter'
 
-import { getGlobPattern } from './config'
+import * as config from './config'
 import { flattenArray, flattenObjectValues } from './util/flatten'
 import { getFilePaths } from './util/fs'
 import { getShebang, isBashShebang } from './util/shebang'
@@ -84,7 +84,8 @@ export default class Analyzer {
     connection: LSP.Connection
     rootPath: string
   }) {
-    const globPattern = getGlobPattern()
+    const globPattern = config.getGlobPattern()
+    const backgroundAnalysisMaxFiles = config.getBackgroundAnalysisMaxFiles()
     connection.console.log(
       `BackgroundAnalysis: resolving glob "${globPattern}" inside "${rootPath}"...`,
     )
@@ -97,6 +98,7 @@ export default class Analyzer {
       filePaths = await getFilePaths({
         globPattern,
         rootPath,
+        maxItems: backgroundAnalysisMaxFiles,
       })
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : error

server/src/config.ts

@@ -1,4 +1,5 @@
 export const DEFAULT_GLOB_PATTERN = '**/*@(.sh|.inc|.bash|.command)'
+export const DEFAULT_BACKGROUND_ANALYSIS_MAX_FILES = 5000
 
 export function getShellcheckPath(): string | null {
   const { SHELLCHECK_PATH } = process.env
@@ -26,3 +27,9 @@ export function getHighlightParsingError(): boolean {
     ? HIGHLIGHT_PARSING_ERRORS === 'true' || HIGHLIGHT_PARSING_ERRORS === '1'
     : false
 }
+
+export function getBackgroundAnalysisMaxFiles(): number {
+  const { BACKGROUND_ANALYSIS_MAX_FILES } = process.env
+  const parsed = parseInt(BACKGROUND_ANALYSIS_MAX_FILES || '', 10)
+  return !isNaN(parsed) ? parsed : DEFAULT_BACKGROUND_ANALYSIS_MAX_FILES
+}

server/src/util/fs.ts

@@ -1,9 +1,9 @@
-import * as glob from 'glob'
-import * as Os from 'os'
+import * as fastGlob from 'fast-glob'
+import * as os from 'os'
 
 // from https://github.com/sindresorhus/untildify/blob/f85a087418aeaa2beb56fe2684fe3b64fc8c588d/index.js#L11
 export function untildify(pathWithTilde: string): string {
-  const homeDirectory = Os.homedir()
+  const homeDirectory = os.homedir()
   return homeDirectory
     ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory)
     : pathWithTilde
@@ -12,21 +12,33 @@ export function untildify(pathWithTilde: string): string {
 export async function getFilePaths({
   globPattern,
   rootPath,
+  maxItems,
 }: {
   globPattern: string
   rootPath: string
+  maxItems: number
 }): Promise<string[]> {
-  return new Promise((resolve, reject) => {
-    glob(
-      globPattern,
-      { cwd: rootPath, nodir: true, absolute: true, strict: false },
-      function (err, files) {
-        if (err) {
-          return reject(err)
-        }
-
-        resolve(files)
-      },
-    )
+  const stream = fastGlob.stream([globPattern], {
+    absolute: true,
+    onlyFiles: true,
+    cwd: rootPath,
+    followSymbolicLinks: true,
+    suppressErrors: true,
   })
+
+  // NOTE: we use a stream here to not block the event loop
+  // and ensure that we stop reading files if the glob returns
+  // too many files.
+  const files = []
+  let i = 0
+  for await (const fileEntry of stream) {
+    if (i >= maxItems) {
+      break
+    }
+
+    files.push(fileEntry.toString())
+    i++
+  }
+
+  return files
 }