feat: split up API Routes + use .nft.json files to make builds fast

packages/runtime/package.json

@@ -14,6 +14,7 @@
     "@netlify/esbuild": "0.14.39",
     "@netlify/functions": "^1.4.0",
     "@netlify/ipx": "^1.4.0",
+    "@vercel/nft": "^0.22.6",
     "@vercel/node-bridge": "^2.1.0",
     "chalk": "^4.1.2",
     "chokidar": "^3.5.3",

packages/runtime/src/helpers/config.ts

@@ -8,6 +8,7 @@ import slash from 'slash'
 
 import { HANDLER_FUNCTION_NAME, IMAGE_FUNCTION_NAME, ODB_FUNCTION_NAME } from '../constants'
 
+import type { APILambda } from './functions'
 import type { RoutesManifest } from './types'
 import { escapeStringRegexp } from './utils'
 
@@ -98,10 +99,12 @@ export const configureHandlerFunctions = async ({
   netlifyConfig,
   publish,
   ignore = [],
+  apiLambdas,
 }: {
   netlifyConfig: NetlifyConfig
   publish: string
   ignore: Array<string>
+  apiLambdas: APILambda[]
 }) => {
   const config = await getRequiredServerFiles(publish)
   const files = config.files || []
@@ -117,7 +120,7 @@ export const configureHandlerFunctions = async ({
     (moduleName) => !hasManuallyAddedModule({ netlifyConfig, moduleName }),
   )
 
-  ;[HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME, '_api_*'].forEach((functionName) => {
+  ;[HANDLER_FUNCTION_NAME, ODB_FUNCTION_NAME].forEach((functionName: string) => {
     netlifyConfig.functions[functionName] ||= { included_files: [], external_node_modules: [] }
     netlifyConfig.functions[functionName].node_bundler = 'nft'
     netlifyConfig.functions[functionName].included_files ||= []
@@ -157,6 +160,14 @@ export const configureHandlerFunctions = async ({
       }
     })
   })
+
+  for (const apiLambda of apiLambdas) {
+    const { functionName, includedFiles } = apiLambda
+    // TODO: add all the nextRoot/wasm/excludedModules stuff from above
+    netlifyConfig.functions[functionName] ||= { included_files: [] }
+    netlifyConfig.functions[functionName].node_bundler = 'none'
+    netlifyConfig.functions[functionName].included_files = includedFiles
+  }
 }
 
 interface BuildHeaderParams {

packages/runtime/src/helpers/files.ts

@@ -364,7 +364,7 @@ export const getDependenciesOfFile = async (file: string) => {
   if (!existsSync(nft)) {
     return []
   }
-  const dependencies = await readJson(nft, 'utf8')
+  const dependencies = (await readJson(nft, 'utf8')) as { files: string[] }
   return dependencies.files.map((dep) => resolve(dirname(file), dep))
 }
 

packages/runtime/src/helpers/functions.ts

@@ -1,11 +1,13 @@
 import type { NetlifyConfig, NetlifyPluginConstants } from '@netlify/build'
+import { nodeFileTrace } from '@vercel/nft'
 import bridgeFile from '@vercel/node-bridge'
 import chalk from 'chalk'
 import destr from 'destr'
-import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON } from 'fs-extra'
+import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON, stat } from 'fs-extra'
 import type { ImageConfigComplete, RemotePattern } from 'next/dist/shared/lib/image-config'
 import { outdent } from 'outdent'
-import { join, relative, resolve } from 'pathe'
+import { join, relative, resolve, dirname } from 'pathe'
+import glob from 'tiny-glob'
 
 import {
   HANDLER_FUNCTION_NAME,
@@ -21,38 +23,47 @@ import { getHandler } from '../templates/getHandler'
 import { getResolverForPages, getResolverForSourceFiles } from '../templates/getPageResolver'
 
 import { ApiConfig, ApiRouteType, extractConfigFromFile, isEdgeConfig } from './analysis'
-import { getSourceFileForPage } from './files'
+import { getSourceFileForPage, getDependenciesOfFile } from './files'
 import { writeFunctionConfiguration } from './functionsMetaData'
+import { pack } from './pack'
 import { getFunctionNameForPage } from './utils'
 
+// TODO, for reviewer: I added my properties here because that was the easiest way,
+// but is it the right spot for it?
 export interface ApiRouteConfig {
+  functionName: string
   route: string
   config: ApiConfig
   compiled: string
+  includedFiles: string[]
+}
+
+export interface APILambda {
+  functionName: string
+  routes: ApiRouteConfig[]
+  includedFiles: string[]
+  type?: ApiRouteType
 }
 
 export const generateFunctions = async (
   { FUNCTIONS_SRC = DEFAULT_FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, PUBLISH_DIR }: NetlifyPluginConstants,
   appDir: string,
-  apiRoutes: Array<ApiRouteConfig>,
+  apiLambdas: APILambda[],
 ): Promise<void> => {
   const publish = resolve(PUBLISH_DIR)
   const functionsDir = resolve(INTERNAL_FUNCTIONS_SRC || FUNCTIONS_SRC)
   const functionDir = join(functionsDir, HANDLER_FUNCTION_NAME)
   const publishDir = relative(functionDir, publish)
 
-  for (const { route, config, compiled } of apiRoutes) {
-    // Don't write a lambda if the runtime is edge
-    if (isEdgeConfig(config.runtime)) {
-      continue
-    }
-    const apiHandlerSource = await getApiHandler({
-      page: route,
-      config,
+  for (const apiLambda of apiLambdas) {
+    const { functionName, routes, type, includedFiles } = apiLambda
+
+    const apiHandlerSource = getApiHandler({
+      schedule: type === ApiRouteType.SCHEDULED ? routes[0].config.schedule : undefined,
       publishDir,
       appDir: relative(functionDir, appDir),
     })
-    const functionName = getFunctionNameForPage(route, config.type === ApiRouteType.BACKGROUND)
+
     await ensureDir(join(functionsDir, functionName))
 
     // write main API handler file
@@ -71,16 +82,25 @@ export const generateFunctions = async (
 
     const resolveSourceFile = (file: string) => join(publish, 'server', file)
 
+    // TODO: this should be unneeded once we use the `none` bundler
     const resolverSource = await getResolverForSourceFiles({
       functionsDir,
       // These extra pages are always included by Next.js
-      sourceFiles: [compiled, 'pages/_app.js', 'pages/_document.js', 'pages/_error.js'].map(resolveSourceFile),
+      sourceFiles: [
+        ...routes.map((route) => route.compiled),
+        'pages/_app.js',
+        'pages/_document.js',
+        'pages/_error.js',
+      ].map(resolveSourceFile),
     })
     await writeFile(join(functionsDir, functionName, 'pages.js'), resolverSource)
+
+    const nfInternalFiles = await glob(join(functionsDir, functionName, '**'))
+    includedFiles.push(...nfInternalFiles)
   }
 
   const writeHandler = async (functionName: string, functionTitle: string, isODB: boolean) => {
-    const handlerSource = await getHandler({ isODB, publishDir, appDir: relative(functionDir, appDir) })
+    const handlerSource = getHandler({ isODB, publishDir, appDir: relative(functionDir, appDir) })
     await ensureDir(join(functionsDir, functionName))
 
     // write main handler file (standard or ODB)
@@ -196,6 +216,121 @@ export const setupImageFunction = async ({
   }
 }
 
+const traceRequiredServerFiles = async (publish: string): Promise<string[]> => {
+  const requiredServerFilesPath = join(publish, 'required-server-files.json')
+  const { files } = (await readJSON(requiredServerFilesPath)) as { files: string[] }
+  files.push(requiredServerFilesPath)
+  return files
+}
+
+const traceNextServer = async (publish: string, baseDir: string): Promise<string[]> => {
+  const nextServerDeps = await getDependenciesOfFile(join(publish, 'next-server.js'))
+
+  // this takes quite a long while, but it's only done once per build.
+  // theoretically, it's only needed once per version of Next.js 🤷
+  const { fileList } = await nodeFileTrace(nextServerDeps, { base: '/' })
+
+  const filtered = [...fileList].filter((f) => {
+    // for some reason, NFT detects /bin/sh. let's not upload that to lambda.
+    if (f === 'bin/sh') return false
+
+    // NFT detects a bunch of large development files that we don't need.
+    if (f.endsWith('.development.js')) return false
+
+    // not needed for API Routes!
+    if (f === 'node_modules/sass/sass.dart.js') return false
+
+    return true
+  })
+
+  return filtered.map((f) => `/${f}`).map((file) => relative(baseDir, file))
+}
+
+export const getAPIPRouteCommonDependencies = async (publish: string, baseDir: string) => {
+  const [requiredServerFiles, nextServerFiles, followRedirectsFiles] = await Promise.all([
+    traceRequiredServerFiles(publish),
+    traceNextServer(publish, baseDir),
+
+    // used by our own bridge.js
+    glob(join(dirname(require.resolve('follow-redirects', { paths: [publish] })), '**', '*')),
+  ])
+
+  return [...requiredServerFiles, ...nextServerFiles, ...followRedirectsFiles]
+}
+
+const sum = (arr: number[]) => arr.reduce((v, current) => v + current, 0)
+
+// TODO: cache results
+const getBundleWeight = async (patterns: string[]) => {
+  const sizes = await Promise.all(
+    patterns.flatMap(async (pattern) => {
+      const files = await glob(pattern)
+      return Promise.all(
+        files.map(async (file) => {
+          const fStat = await stat(file)
+          if (fStat.isFile()) {
+            return fStat.size
+          }
+          return 0
+        }),
+      )
+    }),
+  )
+
+  return sum(sizes.flat(1))
+}
+
+const MB = 1024 * 1024
+
+export const getAPILambdas = async (publish: string, baseDir: string): Promise<APILambda[]> => {
+  console.time('traceCommonDependencies')
+  const commonDependencies = await getAPIPRouteCommonDependencies(publish, baseDir)
+  console.timeEnd('traceCommonDependencies')
+
+  console.time('weighCommonDependencies')
+  const threshold = 50 * MB - (await getBundleWeight(commonDependencies))
+  console.timeEnd('weighCommonDependencies')
+
+  const apiRoutes = await getApiRouteConfigs(publish, baseDir)
+
+  const packFunctions = async (routes: ApiRouteConfig[], type?: ApiRouteType): Promise<APILambda[]> => {
+    const weighedRoutes = await Promise.all(
+      routes.map(async (route) => ({ value: route, weight: await getBundleWeight(route.includedFiles) })),
+    )
+
+    const bins = pack(weighedRoutes, threshold)
+
+    return bins.map((bin, index) => ({
+      functionName: bin.length === 1 ? bin[0].functionName : `api-${index}`,
+      routes: bin,
+      includedFiles: [...commonDependencies, ...routes.flatMap((route) => route.includedFiles)],
+      type,
+    }))
+  }
+
+  const standardFunctions = apiRoutes.filter(
+    (route) =>
+      !isEdgeConfig(route.config.runtime) &&
+      route.config.type !== ApiRouteType.BACKGROUND &&
+      route.config.type !== ApiRouteType.SCHEDULED,
+  )
+  const scheduledFunctions = apiRoutes.filter((route) => route.config.type === ApiRouteType.SCHEDULED)
+  const backgroundFunctions = apiRoutes.filter((route) => route.config.type === ApiRouteType.BACKGROUND)
+
+  const scheduledLambdas: APILambda[] = scheduledFunctions.map((func) => ({
+    functionName: func.functionName,
+    includedFiles: func.includedFiles,
+    routes: [func],
+    type: func.config.type,
+  }))
+
+  const [standardLambdas, backgroundLambdas] = await Promise.all([
+    packFunctions(standardFunctions),
+    packFunctions(backgroundFunctions, ApiRouteType.BACKGROUND),
+  ])
+  return [...standardLambdas, ...backgroundLambdas, ...scheduledLambdas]
+}
+
 /**
  * Look for API routes, and extract the config from the source file.
  */
@@ -210,19 +345,25 @@ export const getApiRouteConfigs = async (publish: string, baseDir: string): Prom
   return await Promise.all(
     apiRoutes.map(async (apiRoute) => {
       const filePath = getSourceFileForPage(apiRoute, [pagesDir, srcPagesDir])
-      return { route: apiRoute, config: await extractConfigFromFile(filePath), compiled: pages[apiRoute] }
-    }),
-  )
-}
+      const config = await extractConfigFromFile(filePath)
 
-/**
- * Looks for extended API routes (background and scheduled functions) and extract the config from the source file.
- */
-export const getExtendedApiRouteConfigs = async (publish: string, baseDir: string): Promise<Array<ApiRouteConfig>> => {
-  const settledApiRoutes = await getApiRouteConfigs(publish, baseDir)
+      const functionName = getFunctionNameForPage(apiRoute, config.type === ApiRouteType.BACKGROUND)
+
+      const compiled = pages[apiRoute]
+      const compiledPath = join(publish, 'server', compiled)
+
+      const routeDependencies = await getDependenciesOfFile(compiledPath)
+      const includedFiles = [compiledPath, ...routeDependencies]
 
-  // We only want to return the API routes that are background or scheduled functions
-  return settledApiRoutes.filter((apiRoute) => apiRoute.config.type !== undefined)
+      return {
+        functionName,
+        route: apiRoute,
+        config,
+        compiled,
+        includedFiles,
+      }
+    }),
+  )
 }
 
 interface FunctionsManifest {

packages/runtime/src/helpers/pack.ts

@@ -0,0 +1,39 @@
+/**
+ * Naïve linear packing algorithm.
+ * Takes items with weights, and packs them into boxes of a given threshold.
+ * If an item weight exceeds the threshold, it is put into a box of its own.
+ *
+ * We're using this to combine many API Routes into fewer Lambda functions.
+ *
+ * This does not compute an optimal solution.
+ * For that, we'd take the full dependency graph into account
+ * and try to pack routes with intersecting dependencies together.
+ * But since most of the lambda bundle consists of node_modules,
+ * that probably won't help much.
+ * In the future, we might think about using some graph-based analysis here!
+ * Too complicated for now.
+ */
+export const pack = <T>(items: { value: T; weight: number }[], threshold: number): T[][] => {
+  const result: T[][] = []
+
+  let currentBox: T[] = []
+  let currentWeight = 0
+
+  const sortedDescending = items.sort((a, b) => b.weight - a.weight)
+  for (const item of sortedDescending) {
+    const fitsInCurrentBox = currentWeight + item.weight <= threshold
+    if (fitsInCurrentBox) {
+      currentBox.push(item.value)
+      currentWeight += item.weight
+    } else {
+      if (currentBox.length !== 0) result.push(currentBox)
+
+      currentBox = [item.value]
+      currentWeight = item.weight
+    }
+  }
+
+  if (currentBox.length !== 0) result.push(currentBox)
+
+  return result
+}

packages/runtime/src/helpers/redirects.ts

@@ -10,7 +10,7 @@ import { HANDLER_FUNCTION_PATH, HIDDEN_PATHS, ODB_FUNCTION_PATH } from '../const
 
 import { isAppDirRoute, loadAppPathRoutesManifest } from './edge'
 import { getMiddleware } from './files'
-import { ApiRouteConfig } from './functions'
+import { APILambda } from './functions'
 import { RoutesManifest } from './types'
 import {
   getApiRewrites,
@@ -262,12 +262,12 @@ export const generateRedirects = async ({
   netlifyConfig,
   nextConfig: { i18n, basePath, trailingSlash, appDir },
   buildId,
-  apiRoutes,
+  apiLambdas,
 }: {
   netlifyConfig: NetlifyConfig
   nextConfig: Pick<NextConfig, 'i18n' | 'basePath' | 'trailingSlash' | 'appDir'>
   buildId: string
-  apiRoutes: Array<ApiRouteConfig>
+  apiLambdas: APILambda[]
 }) => {
   const { dynamicRoutes: prerenderedDynamicRoutes, routes: prerenderedStaticRoutes }: PrerenderManifest =
     await readJSON(join(netlifyConfig.build.publish, 'prerender-manifest.json'))
@@ -285,7 +285,7 @@ export const generateRedirects = async ({
   // This is only used in prod, so dev uses `next dev` directly
   netlifyConfig.redirects.push(
     // API routes always need to be served from the regular function
-    ...getApiRewrites(basePath, apiRoutes),
+    ...getApiRewrites(basePath, apiLambdas),
     // Preview mode gets forced to the function, to bypass pre-rendered pages, but static files need to be skipped
     ...(await getPreviewRewrites({ basePath, appDir })),
   )