From 9fffa23f3fa45635ccc308c039648f801defca37 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 21 Oct 2021 14:36:09 +0100 Subject: [PATCH 1/2] feat: warn if zip is too large, and log the largest files --- package-lock.json | 23 +++++++++++++++++--- package.json | 2 ++ src/helpers/verification.js | 43 +++++++++++++++++++++++++++++++++++-- src/index.js | 7 ++++-- test/index.js | 1 + 5 files changed, 69 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8466142bd6..2c962fc853 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,8 +15,10 @@ "chalk": "^4.1.2", "fs-extra": "^10.0.0", "moize": "^6.1.0", + "node-stream-zip": "^1.15.0", "outdent": "^0.8.0", "pathe": "^0.2.0", + "pretty-bytes": "^5.6.0", "semver": "^7.3.5", "slash": "^3.0.0", "tiny-glob": "^0.2.9" @@ -13677,6 +13679,18 @@ "integrity": "sha512-rB1DUFUNAN4Gn9keO2K1efO35IDK7yKHCdCaIMvFO7yUYmmZYeDjnGKle26G4rwj+LKRQpjyUUvMkPglwGCYNQ==", "dev": true }, + "node_modules/node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "engines": { + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" + } + }, "node_modules/node-version-matches": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/node-version-matches/-/node-version-matches-2.0.1.tgz", @@ -14867,7 +14881,6 @@ "version": "5.6.0", "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==", - "dev": true, "engines": { "node": ">=6" }, @@ -29161,6 +29174,11 @@ "integrity": "sha512-rB1DUFUNAN4Gn9keO2K1efO35IDK7yKHCdCaIMvFO7yUYmmZYeDjnGKle26G4rwj+LKRQpjyUUvMkPglwGCYNQ==", "dev": true }, + "node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==" + }, "node-version-matches": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/node-version-matches/-/node-version-matches-2.0.1.tgz", @@ -30056,8 +30074,7 @@ "pretty-bytes": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", - "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==", - "dev": true + "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==" }, "pretty-format": { "version": "27.3.0", diff --git a/package.json b/package.json index 65f8874175..b08ffbbfc9 100644 --- a/package.json +++ b/package.json @@ -58,8 +58,10 @@ "chalk": "^4.1.2", "fs-extra": "^10.0.0", "moize": "^6.1.0", + "node-stream-zip": "^1.15.0", "outdent": "^0.8.0", "pathe": "^0.2.0", + "pretty-bytes": "^5.6.0", "semver": "^7.3.5", "slash": "^3.0.0", "tiny-glob": "^0.2.9" diff --git a/src/helpers/verification.js b/src/helpers/verification.js index f2c2706515..63401ad152 100644 --- a/src/helpers/verification.js +++ b/src/helpers/verification.js @@ -1,8 +1,11 @@ +const { existsSync, promises } = require('fs') const path = require('path') +const { relative } = require('path') -const { yellowBright, greenBright, blueBright } = require('chalk') -const { existsSync } = require('fs-extra') +const { yellowBright, greenBright, blueBright, redBright } = require('chalk') +const { async: StreamZip } = require('node-stream-zip') const outdent = require('outdent') +const prettyBytes = require('pretty-bytes') const { satisfies } = require('semver') exports.verifyBuildTarget = (target) => { @@ -53,6 +56,42 @@ exports.checkForRootPublish = ({ publish, failBuild }) => { } } +// 50MB, which is the documented max, though the hard max seems to be higher +const LAMBDA_MAX_SIZE = 1024 * 1024 * 50 + +exports.checkZipSize = async (file) => { + const size = await promises.stat(file).then(({ size }) => size) + if (size < LAMBDA_MAX_SIZE) { + return + } + // We don't fail the build, because the actual hard max size is larger so it might still succeed + console.log( + redBright(outdent` + + The function zip ${yellowBright(relative(process.cwd(), file))} size is ${prettyBytes( + size, + )}, which is larger than the maximum supported size of ${prettyBytes(LAMBDA_MAX_SIZE)}. + There are a few reasons this could happen. You may have accidentally bundled a large dependency, or you might have a + large number of pre-rendered pages included. + + `), + ) + const zip = new StreamZip({ file }) + console.log(`Contains ${await zip.entriesCount} files`) + const sortedFiles = Object.values(await zip.entries()).sort((a, b) => b.size - a.size) + + const largest = {} + for (let i = 0; i < 10; i++) { + largest[`${i + 1}`] = { + File: sortedFiles[i].name, + 'Compressed Size': prettyBytes(sortedFiles[i].compressedSize), + 'Uncompressed Size': prettyBytes(sortedFiles[i].size), + } + } + console.log(yellowBright`\n\nThese are the largest files in the zip:`) + console.table(largest) +} + exports.logBetaMessage = () => console.log( greenBright( diff --git a/src/index.js b/src/index.js index 0ff438e780..ca80341af8 100644 --- a/src/index.js +++ b/src/index.js @@ -4,6 +4,7 @@ const { join, relative } = require('path') const { copy, existsSync } = require('fs-extra') +const { ODB_FUNCTION_NAME } = require('./constants') const { restoreCache, saveCache } = require('./helpers/cache') const { getNextConfig, configureHandlerFunctions, generateRedirects } = require('./helpers/config') const { generateFunctions, setupImageFunction, generatePagesResolver } = require('./helpers/functions') @@ -13,6 +14,7 @@ const { verifyBuildTarget, checkForRootPublish, logBetaMessage, + checkZipSize, } = require('./helpers/verification') module.exports = { @@ -65,8 +67,9 @@ module.exports = { }) }, - async onPostBuild({ netlifyConfig, utils: { cache } }) { - return saveCache({ cache, publish: netlifyConfig.build.publish }) + async onPostBuild({ netlifyConfig, utils: { cache }, constants }) { + await saveCache({ cache, publish: netlifyConfig.build.publish }) + await checkZipSize(join(process.cwd(), constants.FUNCTIONS_DIST, `${ODB_FUNCTION_NAME}.zip`)) }, onEnd() { logBetaMessage() diff --git a/test/index.js b/test/index.js index 816d71f7f6..4001225c36 100644 --- a/test/index.js +++ b/test/index.js @@ -14,6 +14,7 @@ const SAMPLE_PROJECT_DIR = `${__dirname}/../demo` const constants = { INTERNAL_FUNCTIONS_SRC: '.netlify/internal-functions', PUBLISH_DIR: '.next', + FUNCTIONS_DIST: '.netlify/functions', } const utils = { build: { From 821797278df596b8c8e002ab8e2ba4a740d40228 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 25 Oct 2021 12:01:10 +0100 Subject: [PATCH 2/2] chore: fix test --- src/helpers/verification.js | 4 ++++ test/index.js | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/helpers/verification.js b/src/helpers/verification.js index 63401ad152..ef9538289b 100644 --- a/src/helpers/verification.js +++ b/src/helpers/verification.js @@ -60,6 +60,10 @@ exports.checkForRootPublish = ({ publish, failBuild }) => { const LAMBDA_MAX_SIZE = 1024 * 1024 * 50 exports.checkZipSize = async (file) => { + if (!existsSync(file)) { + console.warn(`Could not check zip size because ${file} does not exist`) + return + } const size = await promises.stat(file).then(({ size }) => size) if (size < LAMBDA_MAX_SIZE) { return diff --git a/test/index.js b/test/index.js index 4001225c36..aca72c260b 100644 --- a/test/index.js +++ b/test/index.js @@ -227,8 +227,6 @@ describe('onBuild()', () => { describe('onPostBuild', () => { test('saves cache with right paths', async () => { - await useFixture('dist_dir_next_config') - const save = jest.fn() await plugin.onPostBuild({