From a8aa12e032063df5dfce9492f7b397ada0144627 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 16 Jun 2023 13:17:27 +0200 Subject: [PATCH 1/8] feat: use "none bundler" for SSR --- packages/runtime/src/helpers/config.ts | 28 +++-- packages/runtime/src/helpers/flags.ts | 6 + packages/runtime/src/helpers/functions.ts | 119 +++++++++++++++++-- packages/runtime/src/index.ts | 9 +- packages/runtime/src/templates/getHandler.ts | 2 + 5 files changed, 145 insertions(+), 19 deletions(-) diff --git a/packages/runtime/src/helpers/config.ts b/packages/runtime/src/helpers/config.ts index 42198606f9..c6c5ae3886 100644 --- a/packages/runtime/src/helpers/config.ts +++ b/packages/runtime/src/helpers/config.ts @@ -8,7 +8,7 @@ import slash from 'slash' import { HANDLER_FUNCTION_NAME, IMAGE_FUNCTION_NAME, ODB_FUNCTION_NAME } from '../constants' -import type { APILambda } from './functions' +import type { APILambda, SSRLambda } from './functions' import type { RoutesManifest } from './types' import { escapeStringRegexp } from './utils' @@ -107,12 +107,14 @@ export const configureHandlerFunctions = async ({ publish, ignore = [], apiLambdas, + ssrLambdas, splitApiRoutes, }: { netlifyConfig: NetlifyConfig publish: string ignore: Array apiLambdas: APILambda[] + ssrLambdas: SSRLambda[] splitApiRoutes: boolean }) => { const config = await getRequiredServerFiles(publish) @@ -170,17 +172,23 @@ export const configureHandlerFunctions = async ({ }) } - configureFunction(HANDLER_FUNCTION_NAME) - configureFunction(ODB_FUNCTION_NAME) + const configureLambda = (lambda: APILambda) => { + const { functionName, includedFiles } = lambda + netlifyConfig.functions[functionName] ||= { included_files: [] } + netlifyConfig.functions[functionName].node_bundler = 'none' + netlifyConfig.functions[functionName].included_files ||= [] + netlifyConfig.functions[functionName].included_files.push(...includedFiles.map(escapeGlob)) + } + + if (ssrLambdas.length === 0) { + configureFunction(HANDLER_FUNCTION_NAME) + configureFunction(ODB_FUNCTION_NAME) + } else { + ssrLambdas.forEach(configureLambda) + } if (splitApiRoutes) { - for (const apiLambda of apiLambdas) { - const { functionName, includedFiles } = apiLambda - netlifyConfig.functions[functionName] ||= { included_files: [] } - netlifyConfig.functions[functionName].node_bundler = 'none' - netlifyConfig.functions[functionName].included_files ||= [] - netlifyConfig.functions[functionName].included_files.push(...includedFiles.map(escapeGlob)) - } + apiLambdas.forEach(configureLambda) } else { configureFunction('_api_*') } diff --git a/packages/runtime/src/helpers/flags.ts b/packages/runtime/src/helpers/flags.ts index 69f633ee03..9a5f11f9a9 100644 --- a/packages/runtime/src/helpers/flags.ts +++ b/packages/runtime/src/helpers/flags.ts @@ -29,3 +29,9 @@ export const splitApiRoutes = (featureFlags: Record, publish: s return isEnabled } + +export const bundleBasedOnNftFiles = (featureFlags: Record): boolean => { + const isEnabled = destr(process.env.NEXT_BUNDLE_BASED_ON_NFT_FILES) ?? featureFlags.next_bundle_based_on_nft_files ?? false + + return isEnabled +} diff --git a/packages/runtime/src/helpers/functions.ts b/packages/runtime/src/helpers/functions.ts index 98a770b1c3..814dbe1a8e 100644 --- a/packages/runtime/src/helpers/functions.ts +++ b/packages/runtime/src/helpers/functions.ts @@ -3,9 +3,10 @@ import bridgeFile from '@vercel/node-bridge' import chalk from 'chalk' import destr from 'destr' import { copyFile, ensureDir, existsSync, readJSON, writeFile, writeJSON, stat } from 'fs-extra' +import { PrerenderManifest } from 'next/dist/build' import type { ImageConfigComplete, RemotePattern } from 'next/dist/shared/lib/image-config' import { outdent } from 'outdent' -import { join, relative, resolve, dirname } from 'pathe' +import { join, relative, resolve, dirname, basename, extname } from 'pathe' import glob from 'tiny-glob' import { @@ -32,20 +33,27 @@ import { pack } from './pack' import { ApiRouteType } from './types' import { getFunctionNameForPage } from './utils' -export interface ApiRouteConfig { +export interface RouteConfig { functionName: string functionTitle?: string route: string - config: ApiConfig compiled: string includedFiles: string[] } -export interface APILambda { +export interface ApiRouteConfig extends RouteConfig { + config: ApiConfig +} + +export interface SSRLambda { functionName: string functionTitle: string - routes: ApiRouteConfig[] + routes: RouteConfig[] includedFiles: string[] +} + +export interface APILambda extends SSRLambda { + routes: ApiRouteConfig[] type?: ApiRouteType } @@ -53,6 +61,7 @@ export const generateFunctions = async ( { FUNCTIONS_SRC = DEFAULT_FUNCTIONS_SRC, INTERNAL_FUNCTIONS_SRC, PUBLISH_DIR }: NetlifyPluginConstants, appDir: string, apiLambdas: APILambda[], + ssrLambdas: SSRLambda[], ): Promise => { const publish = resolve(PUBLISH_DIR) const functionsDir = resolve(INTERNAL_FUNCTIONS_SRC || FUNCTIONS_SRC) @@ -144,6 +153,12 @@ export const generateFunctions = async ( join(functionsDir, functionName, 'handlerUtils.js'), ) await writeFunctionConfiguration({ functionName, functionTitle, functionsDir }) + + const nfInternalFiles = await glob(join(functionsDir, functionName, '**')) + const lambda = ssrLambdas.find((l) => l.functionName === functionName) + if (lambda) { + lambda.includedFiles.push(...nfInternalFiles) + } } await writeHandler(HANDLER_FUNCTION_NAME, HANDLER_FUNCTION_TITLE, false) @@ -295,13 +310,17 @@ export const traceNPMPackage = async (packageName: string, publish: string) => { } } -export const getAPIPRouteCommonDependencies = async (publish: string) => { +export const getCommonDependencies = async (publish: string) => { const deps = await Promise.all([ traceRequiredServerFiles(publish), traceNextServer(publish), // used by our own bridge.js traceNPMPackage('follow-redirects', publish), + + // using package.json because otherwise, we'd find some /dist/... path + traceNPMPackage('@netlify/functions/package.json', publish), + traceNPMPackage('is-promise', publish), ]) return deps.flat(1) @@ -329,12 +348,98 @@ const getBundleWeight = async (patterns: string[]) => { return sum(sizes.flat(1)) } +const changeExtension = (file: string, extension: string) => { + const base = basename(file, extname(file)) + return join(dirname(file), base + extension) +} + +const getSSRDependencies = async (publish: string): Promise => { + const prerenderManifest: PrerenderManifest = await readJSON(join(publish, 'prerender-manifest.json')) + + return [ + ...Object.entries(prerenderManifest.routes).flatMap(([route, ssgRoute]) => { + if (ssgRoute.initialRevalidateSeconds === false) { + return [] + } + + if (ssgRoute.dataRoute.endsWith('.rsc')) { + return [ + join(publish, 'server', 'app', ssgRoute.dataRoute), + join(publish, 'server', 'app', changeExtension(ssgRoute.dataRoute, '.html')), + ] + } + + const trimmedPath = route === '/' ? 'index' : route.slice(1) + return [ + join(publish, 'server', 'pages', `${trimmedPath}.html`), + join(publish, 'server', 'pages', `${trimmedPath}.json`), + ] + }), + join(publish, '**', '*.html'), + ] +} + +export const getSSRLambdas = async (publish: string): Promise => { + const commonDependencies = await getCommonDependencies(publish) + const ssrRoutes = await getSSRRoutes(publish) + + // TODO: for now, they're the same - but we should separate them + const nonOdbRoutes = ssrRoutes + const odbRoutes = ssrRoutes + + const ssrDependencies = await getSSRDependencies(publish) + + return [ + { + functionName: HANDLER_FUNCTION_NAME, + functionTitle: HANDLER_FUNCTION_TITLE, + includedFiles: [ + ...commonDependencies, + ...ssrDependencies, + ...nonOdbRoutes.flatMap((route) => route.includedFiles), + ], + routes: nonOdbRoutes, + }, + { + functionName: ODB_FUNCTION_NAME, + functionTitle: ODB_FUNCTION_TITLE, + includedFiles: [...commonDependencies, ...ssrDependencies, ...odbRoutes.flatMap((route) => route.includedFiles)], + routes: odbRoutes, + }, + ] +} + +const getSSRRoutes = async (publish: string): Promise => { + const pages = (await readJSON(join(publish, 'server', 'pages-manifest.json'))) as Record + const routes = Object.entries(pages).filter( + ([page, compiled]) => !page.startsWith('/api/') && !compiled.endsWith('.html'), + ) + + return await Promise.all( + routes.map(async ([route, compiled]) => { + const functionName = getFunctionNameForPage(route) + + const compiledPath = join(publish, 'server', compiled) + + const routeDependencies = await getDependenciesOfFile(compiledPath) + const includedFiles = [compiledPath, ...routeDependencies] + + return { + functionName, + route, + compiled, + includedFiles, + } + }), + ) +} + export const getAPILambdas = async ( publish: string, baseDir: string, pageExtensions: string[], ): Promise => { - const commonDependencies = await getAPIPRouteCommonDependencies(publish) + const commonDependencies = await getCommonDependencies(publish) const threshold = LAMBDA_WARNING_SIZE - (await getBundleWeight(commonDependencies)) diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 7f33625ca1..4f4e29ee5c 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -18,7 +18,7 @@ import { import { onPreDev } from './helpers/dev' import { writeEdgeFunctions, loadMiddlewareManifest, cleanupEdgeFunctions } from './helpers/edge' import { moveStaticPages, movePublicFiles, patchNextFiles, removeMetadataFiles } from './helpers/files' -import { splitApiRoutes } from './helpers/flags' +import { bundleBasedOnNftFiles, splitApiRoutes } from './helpers/flags' import { generateFunctions, setupImageFunction, @@ -28,6 +28,7 @@ import { packSingleFunction, getExtendedApiRouteConfigs, APILambda, + getSSRLambdas, } from './helpers/functions' import { generateRedirects, generateStaticRedirects } from './helpers/redirects' import { shouldSkip, isNextAuthInstalled, getCustomImageResponseHeaders, getRemotePatterns } from './helpers/utils' @@ -50,6 +51,7 @@ const plugin: NetlifyPlugin = { cache, }, }) { + console.log("using local version") const { publish } = netlifyConfig.build if (shouldSkip()) { await restoreCache({ cache, publish }) @@ -171,7 +173,9 @@ const plugin: NetlifyPlugin = { extendedRoutes.map(packSingleFunction), ) - await generateFunctions(constants, appDir, apiLambdas) + const ssrLambdas = bundleBasedOnNftFiles(featureFlags) ? await getSSRLambdas(publish) : [] + + await generateFunctions(constants, appDir, apiLambdas, ssrLambdas) await generatePagesResolver(constants) await configureHandlerFunctions({ @@ -179,6 +183,7 @@ const plugin: NetlifyPlugin = { ignore, publish: relative(process.cwd(), publish), apiLambdas, + ssrLambdas, splitApiRoutes: splitApiRoutes(featureFlags, publish), }) diff --git a/packages/runtime/src/templates/getHandler.ts b/packages/runtime/src/templates/getHandler.ts index b76311931a..c38ad01d0f 100644 --- a/packages/runtime/src/templates/getHandler.ts +++ b/packages/runtime/src/templates/getHandler.ts @@ -209,6 +209,8 @@ export const getHandler = ({ throw new Error('Could not find Next.js server') } + process.env.NODE_ENV = 'production'; + const { Server } = require("http"); const { promises } = require("fs"); // We copy the file here rather than requiring from the node module From 74f38d3170d3ddd471f1a8948195dff7c9b6b89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Fri, 16 Jun 2023 14:37:25 +0100 Subject: [PATCH 2/8] chore: bump prerelease version --- packages/runtime/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/package.json b/packages/runtime/package.json index b0175eaf97..7b44aa3ce2 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@netlify/plugin-nextjs", - "version": "4.37.4", + "version": "4.37.5-experimental-no-bundler.0", "description": "Run Next.js seamlessly on Netlify", "main": "index.js", "files": [ From a22074d9db44d4342357f40fdd86325a8787868e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Fri, 16 Jun 2023 15:15:17 +0100 Subject: [PATCH 3/8] chore: fix linting --- .eslintrc.js | 1 + package-lock.json | 2 +- packages/runtime/src/helpers/flags.ts | 3 ++- packages/runtime/src/index.ts | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 7e25abb2e3..cd3cacb08f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -51,6 +51,7 @@ module.exports = { // https://github.com/typescript-eslint/typescript-eslint/issues/2483 'no-shadow': 'off', '@typescript-eslint/no-shadow': 'error', + 'import/max-dependencies': 'off', }, }, { diff --git a/package-lock.json b/package-lock.json index 0827ecf16d..7b286535aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24657,7 +24657,7 @@ }, "packages/runtime": { "name": "@netlify/plugin-nextjs", - "version": "4.37.4", + "version": "4.37.5-experimental-no-bundler.0", "license": "MIT", "dependencies": { "@netlify/esbuild": "0.14.39", diff --git a/packages/runtime/src/helpers/flags.ts b/packages/runtime/src/helpers/flags.ts index 9a5f11f9a9..368895f61f 100644 --- a/packages/runtime/src/helpers/flags.ts +++ b/packages/runtime/src/helpers/flags.ts @@ -31,7 +31,8 @@ export const splitApiRoutes = (featureFlags: Record, publish: s } export const bundleBasedOnNftFiles = (featureFlags: Record): boolean => { - const isEnabled = destr(process.env.NEXT_BUNDLE_BASED_ON_NFT_FILES) ?? featureFlags.next_bundle_based_on_nft_files ?? false + const isEnabled = + destr(process.env.NEXT_BUNDLE_BASED_ON_NFT_FILES) ?? featureFlags.next_bundle_based_on_nft_files ?? false return isEnabled } diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 4f4e29ee5c..511cc6b661 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -51,7 +51,7 @@ const plugin: NetlifyPlugin = { cache, }, }) { - console.log("using local version") + console.log('using local version') const { publish } = netlifyConfig.build if (shouldSkip()) { await restoreCache({ cache, publish }) From 3beb5cb0cce6e20cdd62fbd86b5144e843e532d2 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 20 Jun 2023 17:29:00 +0200 Subject: [PATCH 4/8] chore: enable new path for e2e tests --- demos/canary/netlify.toml | 4 ++++ demos/default/netlify.toml | 2 ++ demos/middleware/netlify.toml | 4 ++++ demos/nx-next-monorepo-demo/netlify.toml | 1 + demos/static-root/netlify.toml | 1 + 5 files changed, 12 insertions(+) diff --git a/demos/canary/netlify.toml b/demos/canary/netlify.toml index 86b1877202..72f2054ba7 100644 --- a/demos/canary/netlify.toml +++ b/demos/canary/netlify.toml @@ -3,6 +3,10 @@ command = "next build" publish = ".next" ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ../..; fi;" +[build.environment] +NEXT_SPLIT_API_ROUTES = "true" +NEXT_BUNDLE_BASED_ON_NFT_FILES = "true" + [[plugins]] package = "@netlify/plugin-nextjs" diff --git a/demos/default/netlify.toml b/demos/default/netlify.toml index ca00bea4c2..61c0e4b2a1 100644 --- a/demos/default/netlify.toml +++ b/demos/default/netlify.toml @@ -10,6 +10,8 @@ CYPRESS_CACHE_FOLDER = "../node_modules/.CypressBinary" # set TERM variable for terminal output TERM = "xterm" NODE_VERSION = "16.15.1" +NEXT_SPLIT_API_ROUTES = "true" +NEXT_BUNDLE_BASED_ON_NFT_FILES = "true" [[headers]] for = "/_next/image/*" diff --git a/demos/middleware/netlify.toml b/demos/middleware/netlify.toml index b7a08292a9..0aac8b1df0 100644 --- a/demos/middleware/netlify.toml +++ b/demos/middleware/netlify.toml @@ -3,6 +3,10 @@ command = "npm run build" publish = ".next" ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ../..; fi;" +[build.environment] +NEXT_SPLIT_API_ROUTES = "true" +NEXT_BUNDLE_BASED_ON_NFT_FILES = "true" + [[plugins]] package = "@netlify/plugin-nextjs" diff --git a/demos/nx-next-monorepo-demo/netlify.toml b/demos/nx-next-monorepo-demo/netlify.toml index bdb5ed59f8..06ab9c9677 100644 --- a/demos/nx-next-monorepo-demo/netlify.toml +++ b/demos/nx-next-monorepo-demo/netlify.toml @@ -5,6 +5,7 @@ ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff [build.environment] NEXT_SPLIT_API_ROUTES = "true" +NEXT_BUNDLE_BASED_ON_NFT_FILES = "true" [dev] command = "npm run start" diff --git a/demos/static-root/netlify.toml b/demos/static-root/netlify.toml index 5755b5b934..72f2054ba7 100644 --- a/demos/static-root/netlify.toml +++ b/demos/static-root/netlify.toml @@ -5,6 +5,7 @@ ignore = "if [ $CACHED_COMMIT_REF == $COMMIT_REF ]; then (exit 1); else git diff [build.environment] NEXT_SPLIT_API_ROUTES = "true" +NEXT_BUNDLE_BASED_ON_NFT_FILES = "true" [[plugins]] package = "@netlify/plugin-nextjs" From 41cb5b4f942e53a7c74c0b199e6858129b44ca8a Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 20 Jun 2023 18:01:03 +0200 Subject: [PATCH 5/8] fix: include static manifest --- packages/runtime/src/helpers/functions.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runtime/src/helpers/functions.ts b/packages/runtime/src/helpers/functions.ts index 814dbe1a8e..29eca05ca0 100644 --- a/packages/runtime/src/helpers/functions.ts +++ b/packages/runtime/src/helpers/functions.ts @@ -376,6 +376,7 @@ const getSSRDependencies = async (publish: string): Promise => { ] }), join(publish, '**', '*.html'), + join(publish, 'static-manifest.json'), ] } From f46e1424b1f85cc5d447ef01fa78ad2578132ada Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 21 Jun 2023 14:25:22 +0200 Subject: [PATCH 6/8] fix: include app-dir files from `app-paths-manifest.json` --- packages/runtime/src/helpers/functions.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/helpers/functions.ts b/packages/runtime/src/helpers/functions.ts index 29eca05ca0..9f15c83de3 100644 --- a/packages/runtime/src/helpers/functions.ts +++ b/packages/runtime/src/helpers/functions.ts @@ -411,11 +411,16 @@ export const getSSRLambdas = async (publish: string): Promise => { } const getSSRRoutes = async (publish: string): Promise => { - const pages = (await readJSON(join(publish, 'server', 'pages-manifest.json'))) as Record - const routes = Object.entries(pages).filter( + const pageManifest = (await readJSON(join(publish, 'server', 'pages-manifest.json'))) as Record + const pageManifestRoutes = Object.entries(pageManifest).filter( ([page, compiled]) => !page.startsWith('/api/') && !compiled.endsWith('.html'), ) + const appPathsManifest: Record = await readJSON(join(publish, 'server', 'app-paths-manifest.json')) + const appRoutes = Object.entries(appPathsManifest) + + const routes = [...pageManifestRoutes, ...appRoutes] + return await Promise.all( routes.map(async ([route, compiled]) => { const functionName = getFunctionNameForPage(route) From 6d3668c93d7c537bc3ee2fae9ce48ee30a184a20 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 21 Jun 2023 14:28:31 +0200 Subject: [PATCH 7/8] fix: add fallback for when no app-paths-manifest was found --- packages/runtime/src/helpers/functions.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/helpers/functions.ts b/packages/runtime/src/helpers/functions.ts index 9f15c83de3..5b85fbdecd 100644 --- a/packages/runtime/src/helpers/functions.ts +++ b/packages/runtime/src/helpers/functions.ts @@ -416,7 +416,9 @@ const getSSRRoutes = async (publish: string): Promise => { ([page, compiled]) => !page.startsWith('/api/') && !compiled.endsWith('.html'), ) - const appPathsManifest: Record = await readJSON(join(publish, 'server', 'app-paths-manifest.json')) + const appPathsManifest: Record = await readJSON( + join(publish, 'server', 'app-paths-manifest.json'), + ).catch(() => ({})) const appRoutes = Object.entries(appPathsManifest) const routes = [...pageManifestRoutes, ...appRoutes] From fc9480ce9cd961e348d13dc9d2e319a68d856dff Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 30 Jun 2023 11:29:17 +0200 Subject: [PATCH 8/8] Update packages/runtime/src/index.ts --- packages/runtime/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 645e5042e4..92e2991843 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -51,7 +51,6 @@ const plugin: NetlifyPlugin = { cache, }, }) { - console.log('using local version') const { publish } = netlifyConfig.build if (shouldSkip()) { await restoreCache({ cache, publish })