From fa4c703f9e297a9e9de2f67a8e41d7fbbd6f2a2a Mon Sep 17 00:00:00 2001 From: Mike Wright Date: Fri, 3 Jan 2020 16:24:09 -0700 Subject: [PATCH 1/4] feat: refactor page rendering to use node worker threads --- .../@vuepress/core/lib/node/build/index.js | 173 ++++++------------ .../@vuepress/core/lib/node/build/worker.js | 128 +++++++++++++ 2 files changed, 183 insertions(+), 118 deletions(-) create mode 100644 packages/@vuepress/core/lib/node/build/worker.js diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index 19ffc11e6d..4244e77e67 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -3,13 +3,14 @@ const EventEmitter = require('events').EventEmitter const webpack = require('webpack') const readline = require('readline') -const escape = require('escape-html') +const { Worker } = require('worker_threads') +const threads = 8 +const verbose = false const { chalk, fs, path, logger, env, performance } = require('@vuepress/shared-utils') const createClientConfig = require('../webpack/createClientConfig') const createServerConfig = require('../webpack/createServerConfig') -const { createBundleRenderer } = require('vue-server-renderer') -const { normalizeHeadTag, applyUserWebpackConfig } = require('../util/index') +const { applyUserWebpackConfig } = require('../util/index') /** * Expose Build Process Class. @@ -50,6 +51,7 @@ module.exports = class Build extends EventEmitter { async render () { // compile! + performance.start() const stats = await compile([this.clientConfig, this.serverConfig]) const serverBundle = require(path.resolve(this.outDir, 'manifest/server.json')) const clientManifest = require(path.resolve(this.outDir, 'manifest/client.json')) @@ -66,44 +68,66 @@ module.exports = class Build extends EventEmitter { await workaroundEmptyStyleChunk(stats, this.outDir) } - // create server renderer using built manifests - this.renderer = createBundleRenderer(serverBundle, { - clientManifest, - runInNewContext: false, - inject: false, - shouldPrefetch: this.context.siteConfig.shouldPrefetch || (() => true), - template: await fs.readFile(this.context.ssrTemplate, 'utf-8') - }) - - // pre-render head tags from user config - this.userHeadTags = (this.context.siteConfig.head || []) - .map(renderHeadTag) - .join('\n ') - // if the user does not have a custom 404.md, generate the theme's default if (!this.context.pages.some(p => p.path === '/404.html')) { - this.context.addPage({ path: '/404.html' }) + await this.context.addPage({ path: '/404.html' }) } // render pages logger.wait('Rendering static HTML...') - + let activeWorkers = 0 const pagePaths = [] - for (const page of this.context.pages) { - pagePaths.push(await this.renderPage(page)) - } + const pagesPerThread = this.context.pages.length / threads + + for (let i = 0; i < threads; i++) { + const startIndex = i * pagesPerThread + const endIndex = (startIndex + pagesPerThread) > this.context.pages.length ? this.context.pages.length + 1 : startIndex + pagesPerThread + const pageData = this.context.pages.slice(startIndex, endIndex) + const pages = pageData.map(p => ({ path: p.path, frontmatter: JSON.stringify(p.frontmatter) })) + + const payload = { + clientManifest: JSON.stringify(clientManifest), + outDir: this.outDir, + pages: Buffer.from(JSON.stringify(pages)), + serverBundle: JSON.stringify(serverBundle), + siteConfig: JSON.stringify(this.context.siteConfig), + ssrTemplate: JSON.stringify(this.context.ssrTemplate), + verbose: verbose, + workerNumber: i + } - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) + const worker = new Worker(path.join(__dirname, './worker.js')) + worker.postMessage(payload) + activeWorkers++ + worker.on('message', (paths) => { + pagePaths.concat(paths) + }) + worker.on('error', (error) => { + console.error( + logger.error(chalk.red(`Worker #${i} sent error: ${error}\n\n${error.stack}\n`), false) + ) + }) + worker.on('exit', (code) => { + activeWorkers-- + if (code === 0) { + logger.success(`Worker ${i} completed successfully.`) + } else { + logger.error(chalk.red(`Worker #${i} sent exit code: ${code}`), false) + } + if (activeWorkers === 0) { + // DONE. + readline.clearLine(process.stdout, 0) + readline.cursorTo(process.stdout, 0) + const relativeDir = path.relative(this.context.cwd, this.outDir) + logger.success(`Generated static files in ${chalk.cyan(relativeDir)}.`) + const { duration } = performance.stop() + logger.success(`It took a total of ${chalk.cyan(`${duration}ms`)} to run the ${chalk.cyan('vuepress build')}.`) + console.log() + } + }) + } await this.context.pluginAPI.applyAsyncOption('generated', pagePaths) - - // DONE. - const relativeDir = path.relative(this.context.cwd, this.outDir) - logger.success(`Generated static files in ${chalk.cyan(relativeDir)}.`) - const { duration } = performance.stop() - logger.developer(`It took a total of ${chalk.cyan(`${duration}ms`)} to run the ${chalk.cyan('vuepress build')}.`) - console.log() } /** @@ -122,47 +146,6 @@ module.exports = class Build extends EventEmitter { this.serverConfig = applyUserWebpackConfig(userConfig, this.serverConfig, true) } } - - /** - * Render page - * - * @param {Page} page - * @returns {Promise} - * @api private - */ - - async renderPage (page) { - const pagePath = decodeURIComponent(page.path) - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) - process.stdout.write(`Rendering page: ${pagePath}`) - - // #565 Avoid duplicate description meta at SSR. - const meta = (page.frontmatter && page.frontmatter.meta || []).filter(item => item.name !== 'description') - const pageMeta = renderPageMeta(meta) - - const context = { - url: page.path, - userHeadTags: this.userHeadTags, - pageMeta, - title: 'VuePress', - lang: 'en', - description: '' - } - - let html - try { - html = await this.renderer.renderToString(context) - } catch (e) { - console.error(logger.error(chalk.red(`Error rendering ${pagePath}:`), false)) - throw e - } - const filename = pagePath.replace(/\/$/, '/index.html').replace(/^\//, '') - const filePath = path.resolve(this.outDir, filename) - await fs.ensureDir(path.dirname(filePath)) - await fs.writeFile(filePath, html) - return filePath - } } /** @@ -195,52 +178,6 @@ function compile (config) { }) } -/** - * Render head tag - * - * @param {Object} tag - * @returns {string} - */ - -function renderHeadTag (tag) { - const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag) - return `<${tagName}${renderAttrs(attributes)}>${innerHTML}${closeTag ? `` : ``}` -} - -/** - * Render html attributes - * - * @param {Object} attrs - * @returns {string} - */ - -function renderAttrs (attrs = {}) { - const keys = Object.keys(attrs) - if (keys.length) { - return ' ' + keys.map(name => `${name}="${escape(attrs[name])}"`).join(' ') - } else { - return '' - } -} - -/** - * Render meta tags - * - * @param {Array} meta - * @returns {Array} - */ - -function renderPageMeta (meta) { - if (!meta) return '' - return meta.map(m => { - let res = ` { - res += ` ${key}="${escape(m[key])}"` - }) - return res + `>` - }).join('') -} - /** * find and remove empty style chunk caused by * https://github.com/webpack-contrib/mini-css-extract-plugin/issues/85 diff --git a/packages/@vuepress/core/lib/node/build/worker.js b/packages/@vuepress/core/lib/node/build/worker.js new file mode 100644 index 0000000000..597ec44002 --- /dev/null +++ b/packages/@vuepress/core/lib/node/build/worker.js @@ -0,0 +1,128 @@ +const { parentPort } = require('worker_threads') +const escape = require('escape-html') +const readline = require('readline') +const { chalk, fs, path, logger } = require('@vuepress/shared-utils') +const { createBundleRenderer } = require('vue-server-renderer') +const { normalizeHeadTag } = require('../util/index') + +/** + * Worker file for HTML page rendering + * + * @param {number} workerNumber + * @param {Array} pages + * @returns {Promise} + * @api private + */ + +parentPort.once('message', async (payload) => { + const siteConfig = JSON.parse(payload.siteConfig) + const ssrTemplate = JSON.parse(payload.ssrTemplate) + + // create server renderer using built manifests + const renderer = createBundleRenderer(JSON.parse(payload.serverBundle), { + clientManifest: JSON.parse(payload.clientManifest), + runInNewContext: false, + inject: false, + shouldPrefetch: siteConfig.shouldPrefetch || (() => true), + template: await fs.readFile(ssrTemplate, 'utf-8') + }) + + // pre-render head tags from user config + const userHeadTags = (siteConfig.head || []) + .map(renderHeadTag) + .join('\n ') + + const pages = JSON.parse(Buffer.from(payload.pages)) + process.stdout.write(`Worker #${payload.workerNumber} beginning rendering of ${pages.length} pages\n`) + const filePaths = [] + for (const page of pages) { + const pagePath = decodeURIComponent(page.path) + readline.clearLine(process.stdout, 0) + readline.cursorTo(process.stdout, 0) + if (payload.verbose) { + process.stdout.write(`Worker #${payload.workerNumber} rendering page: ${pagePath}\n`) + } + + // #565 Avoid duplicate description meta at SSR. + const meta = ((page.frontmatter && page.frontmatter.meta) || []).filter( + item => item.name !== 'description' + ) + const pageMeta = renderPageMeta(meta) + + const context = { + url: page.path, + userHeadTags: userHeadTags, + pageMeta, + title: 'VuePress', + lang: 'en', + description: '' + } + + let html + try { + html = await renderer.renderToString(context) + } catch (e) { + console.error( + logger.error(chalk.red(`Worker #${payload.workerNumber} error rendering ${pagePath}:\n`), false) + ) + throw e + } finally { + const filename = pagePath.replace(/\/$/, '/index.html').replace(/^\//, '') + const filePath = path.resolve(payload.outDir, filename) + await fs.ensureDir(path.dirname(filePath)) + await fs.writeFile(filePath, html) + filePaths.push(filePath) + } + } + parentPort.postMessage({ filePaths }) +}) + +/** + * Render html attributes + * + * @param {Object} attrs + * @returns {string} + */ + +function renderAttrs (attrs = {}) { + const keys = Object.keys(attrs) + if (keys.length) { + return ' ' + keys.map(name => `${name}="${escape(attrs[name])}"`).join(' ') + } else { + return '' + } +} + +/** + * Render head tag + * + * @param {Object} tag + * @returns {string} + */ + +function renderHeadTag (tag) { + const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag) + return `<${tagName}${renderAttrs(attributes)}>${innerHTML}${ + closeTag ? `` : `` + }` +} + +/** + * Render meta tags + * + * @param {Array} meta + * @returns {Array} + */ + +function renderPageMeta (meta) { + if (!meta) return '' + return meta + .map(m => { + let res = ` { + res += ` ${key}="${escape(m[key])}"` + }) + return res + `>` + }) + .join('') +} From 788b9dd535afdeba926a20db228e90a01daa15c2 Mon Sep 17 00:00:00 2001 From: Mike Wright Date: Fri, 3 Jan 2020 16:24:09 -0700 Subject: [PATCH 2/4] feat: refactor page rendering to use node worker threads --- .../@vuepress/core/lib/node/build/index.js | 348 +++++++++--------- .../@vuepress/core/lib/node/build/worker.js | 128 +++++++ 2 files changed, 292 insertions(+), 184 deletions(-) create mode 100644 packages/@vuepress/core/lib/node/build/worker.js diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index 912eca8c23..e2dfa624d3 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -1,26 +1,33 @@ -'use strict' - -const EventEmitter = require('events').EventEmitter -const webpack = require('webpack') -const readline = require('readline') -const escape = require('escape-html') - -const { chalk, fs, path, logger, env, performance } = require('@vuepress/shared-utils') -const createClientConfig = require('../webpack/createClientConfig') -const createServerConfig = require('../webpack/createServerConfig') -const { createBundleRenderer } = require('vue-server-renderer') -const { normalizeHeadTag, applyUserWebpackConfig } = require('../util/index') -const { version } = require('../../../package') +"use strict"; + +const EventEmitter = require("events").EventEmitter; +const webpack = require("webpack"); +const readline = require("readline"); +const { Worker } = require("worker_threads"); +const threads = 8; +const verbose = false; + +const { + chalk, + fs, + path, + logger, + env, + performance +} = require("@vuepress/shared-utils"); +const createClientConfig = require("../webpack/createClientConfig"); +const createServerConfig = require("../webpack/createServerConfig"); +const { applyUserWebpackConfig } = require("../util/index"); /** * Expose Build Process Class. */ module.exports = class Build extends EventEmitter { - constructor (context) { - super() - this.context = context - this.outDir = this.context.outDir + constructor(context) { + super(); + this.context = context; + this.outDir = this.context.outDir; } /** @@ -31,15 +38,17 @@ module.exports = class Build extends EventEmitter { * @api public */ - async process () { + async process() { if (this.context.cwd === this.outDir) { - throw new Error('Unexpected option: "outDir" cannot be set to the current working directory') + throw new Error( + 'Unexpected option: "outDir" cannot be set to the current working directory' + ); } - this.context.resolveCacheLoaderOptions() - await fs.emptyDir(this.outDir) - logger.debug('Dist directory: ' + chalk.gray(this.outDir)) - this.prepareWebpackConfig() + this.context.resolveCacheLoaderOptions(); + await fs.emptyDir(this.outDir); + logger.debug("Dist directory: " + chalk.gray(this.outDir)); + this.prepareWebpackConfig(); } /** @@ -49,62 +58,113 @@ module.exports = class Build extends EventEmitter { * @api public */ - async render () { + async render() { // compile! - const stats = await compile([this.clientConfig, this.serverConfig]) - const serverBundle = require(path.resolve(this.outDir, 'manifest/server.json')) - const clientManifest = require(path.resolve(this.outDir, 'manifest/client.json')) + performance.start(); + const stats = await compile([this.clientConfig, this.serverConfig]); + const serverBundle = require(path.resolve( + this.outDir, + "manifest/server.json" + )); + const clientManifest = require(path.resolve( + this.outDir, + "manifest/client.json" + )); // remove manifests after loading them. - await fs.remove(path.resolve(this.outDir, 'manifest')) + await fs.remove(path.resolve(this.outDir, "manifest")); // ref: https://github.com/vuejs/vuepress/issues/1367 - if (!this.clientConfig.devtool && (!this.clientConfig.plugins - || !this.clientConfig.plugins.some(p => - p instanceof webpack.SourceMapDevToolPlugin - || p instanceof webpack.EvalSourceMapDevToolPlugin - ))) { - await workaroundEmptyStyleChunk(stats, this.outDir) + if ( + !this.clientConfig.devtool && + (!this.clientConfig.plugins || + !this.clientConfig.plugins.some( + p => + p instanceof webpack.SourceMapDevToolPlugin || + p instanceof webpack.EvalSourceMapDevToolPlugin + )) + ) { + await workaroundEmptyStyleChunk(stats, this.outDir); } - // create server renderer using built manifests - this.renderer = createBundleRenderer(serverBundle, { - clientManifest, - runInNewContext: false, - inject: false, - shouldPrefetch: this.context.siteConfig.shouldPrefetch || (() => true), - template: await fs.readFile(this.context.ssrTemplate, 'utf-8') - }) - - // pre-render head tags from user config - this.userHeadTags = (this.context.siteConfig.head || []) - .map(renderHeadTag) - .join('\n ') - // if the user does not have a custom 404.md, generate the theme's default - if (!this.context.pages.some(p => p.path === '/404.html')) { - this.context.addPage({ path: '/404.html' }) + if (!this.context.pages.some(p => p.path === "/404.html")) { + await this.context.addPage({ path: "/404.html" }); } // render pages - logger.wait('Rendering static HTML...') - - const pagePaths = [] - for (const page of this.context.pages) { - pagePaths.push(await this.renderPage(page)) + logger.wait("Rendering static HTML..."); + let activeWorkers = 0; + const pagePaths = []; + const pagesPerThread = this.context.pages.length / threads; + + for (let i = 0; i < threads; i++) { + const startIndex = i * pagesPerThread; + const endIndex = + startIndex + pagesPerThread > this.context.pages.length + ? this.context.pages.length + 1 + : startIndex + pagesPerThread; + const pageData = this.context.pages.slice(startIndex, endIndex); + const pages = pageData.map(p => ({ + path: p.path, + frontmatter: JSON.stringify(p.frontmatter) + })); + + const payload = { + clientManifest: JSON.stringify(clientManifest), + outDir: this.outDir, + pages: Buffer.from(JSON.stringify(pages)), + serverBundle: JSON.stringify(serverBundle), + siteConfig: JSON.stringify(this.context.siteConfig), + ssrTemplate: JSON.stringify(this.context.ssrTemplate), + verbose: verbose, + workerNumber: i + }; + + const worker = new Worker(path.join(__dirname, "./worker.js")); + worker.postMessage(payload); + activeWorkers++; + worker.on("message", paths => { + pagePaths.concat(paths); + }); + worker.on("error", error => { + console.error( + logger.error( + chalk.red(`Worker #${i} sent error: ${error}\n\n${error.stack}\n`), + false + ) + ); + }); + worker.on("exit", code => { + activeWorkers--; + if (code === 0) { + logger.success(`Worker ${i} completed successfully.`); + } else { + logger.error( + chalk.red(`Worker #${i} sent exit code: ${code}`), + false + ); + } + if (activeWorkers === 0) { + // DONE. + readline.clearLine(process.stdout, 0); + readline.cursorTo(process.stdout, 0); + const relativeDir = path.relative(this.context.cwd, this.outDir); + logger.success( + `Generated static files in ${chalk.cyan(relativeDir)}.` + ); + const { duration } = performance.stop(); + logger.success( + `It took a total of ${chalk.cyan( + `${duration}ms` + )} to run the ${chalk.cyan("vuepress build")}.` + ); + console.log(); + } + }); } - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) - - await this.context.pluginAPI.applyAsyncOption('generated', pagePaths) - - // DONE. - const relativeDir = path.relative(this.context.cwd, this.outDir) - logger.success(`Generated static files in ${chalk.cyan(relativeDir)}.`) - const { duration } = performance.stop() - logger.developer(`It took a total of ${chalk.cyan(`${duration}ms`)} to run the ${chalk.cyan('vuepress build')}.`) - console.log() + await this.context.pluginAPI.applyAsyncOption("generated", pagePaths); } /** @@ -113,59 +173,25 @@ module.exports = class Build extends EventEmitter { * @api private */ - prepareWebpackConfig () { - this.clientConfig = createClientConfig(this.context).toConfig() - this.serverConfig = createServerConfig(this.context).toConfig() + prepareWebpackConfig() { + this.clientConfig = createClientConfig(this.context).toConfig(); + this.serverConfig = createServerConfig(this.context).toConfig(); - const userConfig = this.context.siteConfig.configureWebpack + const userConfig = this.context.siteConfig.configureWebpack; if (userConfig) { - this.clientConfig = applyUserWebpackConfig(userConfig, this.clientConfig, false) - this.serverConfig = applyUserWebpackConfig(userConfig, this.serverConfig, true) - } - } - - /** - * Render page - * - * @param {Page} page - * @returns {Promise} - * @api private - */ - - async renderPage (page) { - const pagePath = decodeURIComponent(page.path) - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) - process.stdout.write(`Rendering page: ${pagePath}`) - - // #565 Avoid duplicate description meta at SSR. - const meta = (page.frontmatter && page.frontmatter.meta || []).filter(item => item.name !== 'description') - const pageMeta = renderPageMeta(meta) - - const context = { - url: page.path, - userHeadTags: this.userHeadTags, - pageMeta, - title: 'VuePress', - lang: 'en', - description: '', - version + this.clientConfig = applyUserWebpackConfig( + userConfig, + this.clientConfig, + false + ); + this.serverConfig = applyUserWebpackConfig( + userConfig, + this.serverConfig, + true + ); } - - let html - try { - html = await this.renderer.renderToString(context) - } catch (e) { - console.error(logger.error(chalk.red(`Error rendering ${pagePath}:`), false)) - throw e - } - const filename = pagePath.replace(/\/$/, '/index.html').replace(/^\//, '') - const filePath = path.resolve(this.outDir, filename) - await fs.ensureDir(path.dirname(filePath)) - await fs.writeFile(filePath, html) - return filePath } -} +}; /** * Compile a webpack application and return stats json. @@ -174,73 +200,27 @@ module.exports = class Build extends EventEmitter { * @returns {Promise} */ -function compile (config) { +function compile(config) { return new Promise((resolve, reject) => { webpack(config, (err, stats) => { if (err) { - return reject(err) + return reject(err); } if (stats.hasErrors()) { stats.toJson().errors.forEach(err => { - console.error(err) - }) - reject(new Error(`Failed to compile with errors.`)) - return + console.error(err); + }); + reject(new Error(`Failed to compile with errors.`)); + return; } if (env.isDebug && stats.hasWarnings()) { stats.toJson().warnings.forEach(warning => { - console.warn(warning) - }) + console.warn(warning); + }); } - resolve(stats.toJson({ modules: false })) - }) - }) -} - -/** - * Render head tag - * - * @param {Object} tag - * @returns {string} - */ - -function renderHeadTag (tag) { - const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag) - return `<${tagName}${renderAttrs(attributes)}>${innerHTML}${closeTag ? `` : ``}` -} - -/** - * Render html attributes - * - * @param {Object} attrs - * @returns {string} - */ - -function renderAttrs (attrs = {}) { - const keys = Object.keys(attrs) - if (keys.length) { - return ' ' + keys.map(name => `${name}="${escape(attrs[name])}"`).join(' ') - } else { - return '' - } -} - -/** - * Render meta tags - * - * @param {Array} meta - * @returns {Array} - */ - -function renderPageMeta (meta) { - if (!meta) return '' - return meta.map(m => { - let res = ` { - res += ` ${key}="${escape(m[key])}"` - }) - return res + `>` - }).join('') + resolve(stats.toJson({ modules: false })); + }); + }); } /** @@ -253,20 +233,20 @@ function renderPageMeta (meta) { * @returns {Promise} */ -async function workaroundEmptyStyleChunk (stats, outDir) { +async function workaroundEmptyStyleChunk(stats, outDir) { const styleChunk = stats.children[0].assets.find(a => { - return /styles\.\w{8}\.js$/.test(a.name) - }) - if (!styleChunk) return - const styleChunkPath = path.resolve(outDir, styleChunk.name) - const styleChunkContent = await fs.readFile(styleChunkPath, 'utf-8') - await fs.remove(styleChunkPath) + return /styles\.\w{8}\.js$/.test(a.name); + }); + if (!styleChunk) return; + const styleChunkPath = path.resolve(outDir, styleChunk.name); + const styleChunkContent = await fs.readFile(styleChunkPath, "utf-8"); + await fs.remove(styleChunkPath); // prepend it to app.js. // this is necessary for the webpack runtime to work properly. const appChunk = stats.children[0].assets.find(a => { - return /app\.\w{8}\.js$/.test(a.name) - }) - const appChunkPath = path.resolve(outDir, appChunk.name) - const appChunkContent = await fs.readFile(appChunkPath, 'utf-8') - await fs.writeFile(appChunkPath, styleChunkContent + appChunkContent) + return /app\.\w{8}\.js$/.test(a.name); + }); + const appChunkPath = path.resolve(outDir, appChunk.name); + const appChunkContent = await fs.readFile(appChunkPath, "utf-8"); + await fs.writeFile(appChunkPath, styleChunkContent + appChunkContent); } diff --git a/packages/@vuepress/core/lib/node/build/worker.js b/packages/@vuepress/core/lib/node/build/worker.js new file mode 100644 index 0000000000..597ec44002 --- /dev/null +++ b/packages/@vuepress/core/lib/node/build/worker.js @@ -0,0 +1,128 @@ +const { parentPort } = require('worker_threads') +const escape = require('escape-html') +const readline = require('readline') +const { chalk, fs, path, logger } = require('@vuepress/shared-utils') +const { createBundleRenderer } = require('vue-server-renderer') +const { normalizeHeadTag } = require('../util/index') + +/** + * Worker file for HTML page rendering + * + * @param {number} workerNumber + * @param {Array} pages + * @returns {Promise} + * @api private + */ + +parentPort.once('message', async (payload) => { + const siteConfig = JSON.parse(payload.siteConfig) + const ssrTemplate = JSON.parse(payload.ssrTemplate) + + // create server renderer using built manifests + const renderer = createBundleRenderer(JSON.parse(payload.serverBundle), { + clientManifest: JSON.parse(payload.clientManifest), + runInNewContext: false, + inject: false, + shouldPrefetch: siteConfig.shouldPrefetch || (() => true), + template: await fs.readFile(ssrTemplate, 'utf-8') + }) + + // pre-render head tags from user config + const userHeadTags = (siteConfig.head || []) + .map(renderHeadTag) + .join('\n ') + + const pages = JSON.parse(Buffer.from(payload.pages)) + process.stdout.write(`Worker #${payload.workerNumber} beginning rendering of ${pages.length} pages\n`) + const filePaths = [] + for (const page of pages) { + const pagePath = decodeURIComponent(page.path) + readline.clearLine(process.stdout, 0) + readline.cursorTo(process.stdout, 0) + if (payload.verbose) { + process.stdout.write(`Worker #${payload.workerNumber} rendering page: ${pagePath}\n`) + } + + // #565 Avoid duplicate description meta at SSR. + const meta = ((page.frontmatter && page.frontmatter.meta) || []).filter( + item => item.name !== 'description' + ) + const pageMeta = renderPageMeta(meta) + + const context = { + url: page.path, + userHeadTags: userHeadTags, + pageMeta, + title: 'VuePress', + lang: 'en', + description: '' + } + + let html + try { + html = await renderer.renderToString(context) + } catch (e) { + console.error( + logger.error(chalk.red(`Worker #${payload.workerNumber} error rendering ${pagePath}:\n`), false) + ) + throw e + } finally { + const filename = pagePath.replace(/\/$/, '/index.html').replace(/^\//, '') + const filePath = path.resolve(payload.outDir, filename) + await fs.ensureDir(path.dirname(filePath)) + await fs.writeFile(filePath, html) + filePaths.push(filePath) + } + } + parentPort.postMessage({ filePaths }) +}) + +/** + * Render html attributes + * + * @param {Object} attrs + * @returns {string} + */ + +function renderAttrs (attrs = {}) { + const keys = Object.keys(attrs) + if (keys.length) { + return ' ' + keys.map(name => `${name}="${escape(attrs[name])}"`).join(' ') + } else { + return '' + } +} + +/** + * Render head tag + * + * @param {Object} tag + * @returns {string} + */ + +function renderHeadTag (tag) { + const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag) + return `<${tagName}${renderAttrs(attributes)}>${innerHTML}${ + closeTag ? `` : `` + }` +} + +/** + * Render meta tags + * + * @param {Array} meta + * @returns {Array} + */ + +function renderPageMeta (meta) { + if (!meta) return '' + return meta + .map(m => { + let res = ` { + res += ` ${key}="${escape(m[key])}"` + }) + return res + `>` + }) + .join('') +} From 05d61c63230fd018853129b0db5632bb74f5a528 Mon Sep 17 00:00:00 2001 From: Mike Wright Date: Sat, 15 Feb 2020 00:53:04 -0700 Subject: [PATCH 3/4] feat: refactor page rendering to use node worker threads - Make worker thread count a cli option for build - Update docs - Remove verbose mode, use existing silent option - Make worker thread logger respect silent --- .../@vuepress/core/lib/node/build/index.js | 207 +++++++++--------- .../@vuepress/core/lib/node/build/worker.js | 119 +++++----- packages/docs/docs/api/cli.md | 3 + packages/vuepress/lib/registerCoreCommands.js | 5 +- 4 files changed, 161 insertions(+), 173 deletions(-) diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index c282f14e45..c661928d3e 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -1,11 +1,9 @@ -"use strict"; +`use strict` -const EventEmitter = require("events").EventEmitter; -const webpack = require("webpack"); -const readline = require("readline"); -const { Worker } = require("worker_threads"); -const threads = 8; -const verbose = false; +const EventEmitter = require('events').EventEmitter +const webpack = require('webpack') +const readline = require('readline') +const { Worker } = require('worker_threads') const { chalk, @@ -14,21 +12,20 @@ const { logger, env, performance -} = require("@vuepress/shared-utils"); -const createClientConfig = require("../webpack/createClientConfig"); -const createServerConfig = require("../webpack/createServerConfig"); -const { applyUserWebpackConfig } = require("../util/index"); +} = require('@vuepress/shared-utils') +const createClientConfig = require('../webpack/createClientConfig') +const createServerConfig = require('../webpack/createServerConfig') +const { applyUserWebpackConfig } = require('../util/index') /** * Expose Build Process Class. */ module.exports = class Build extends EventEmitter { - constructor(context) { - super(); - process.env.NODE_ENV = "production"; - this.context = context; - this.outDir = this.context.outDir; + constructor (context) { + super() + this.context = context + this.outDir = this.context.outDir } /** @@ -39,17 +36,17 @@ module.exports = class Build extends EventEmitter { * @api public */ - async process() { + async process () { if (this.context.cwd === this.outDir) { throw new Error( 'Unexpected option: "outDir" cannot be set to the current working directory' - ); + ) } - this.context.resolveCacheLoaderOptions(); - await fs.emptyDir(this.outDir); - logger.debug("Dist directory: " + chalk.gray(this.outDir)); - this.prepareWebpackConfig(); + this.context.resolveCacheLoaderOptions() + await fs.emptyDir(this.outDir) + logger.debug('Dist directory: ' + chalk.gray(this.outDir)) + this.prepareWebpackConfig() } /** @@ -59,57 +56,57 @@ module.exports = class Build extends EventEmitter { * @api public */ - async render() { + async render () { // compile! - performance.start(); - const stats = await compile([this.clientConfig, this.serverConfig]); + performance.start() + const stats = await compile([this.clientConfig, this.serverConfig]) const serverBundle = require(path.resolve( this.outDir, - "manifest/server.json" - )); + 'manifest/server.json' + )) const clientManifest = require(path.resolve( this.outDir, - "manifest/client.json" - )); + 'manifest/client.json' + )) // remove manifests after loading them. - await fs.remove(path.resolve(this.outDir, "manifest")); + await fs.remove(path.resolve(this.outDir, 'manifest')) // ref: https://github.com/vuejs/vuepress/issues/1367 if ( - !this.clientConfig.devtool && - (!this.clientConfig.plugins || - !this.clientConfig.plugins.some( + !this.clientConfig.devtool + && (!this.clientConfig.plugins + || !this.clientConfig.plugins.some( p => - p instanceof webpack.SourceMapDevToolPlugin || - p instanceof webpack.EvalSourceMapDevToolPlugin + p instanceof webpack.SourceMapDevToolPlugin + || p instanceof webpack.EvalSourceMapDevToolPlugin )) ) { - await workaroundEmptyStyleChunk(stats, this.outDir); + await workaroundEmptyStyleChunk(stats, this.outDir) } // if the user does not have a custom 404.md, generate the theme's default - if (!this.context.pages.some(p => p.path === "/404.html")) { - this.context.addPage({ path: "/404.html" }); + if (!this.context.pages.some(p => p.path === '/404.html')) { + this.context.addPage({ path: '/404.html' }) } // render pages - logger.wait("Rendering static HTML..."); + logger.wait('Rendering static HTML...') - let activeWorkers = 0; - const pagePaths = []; - const pagesPerThread = this.context.pages.length / threads; + let activeWorkers = 0 + const pagePaths = [] + const pagesPerThread = this.context.pages.length / env.workerThreads - for (let i = 0; i < threads; i++) { - const startIndex = i * pagesPerThread; + for (let workerNumber = 0; workerNumber < env.workerThreads; workerNumber++) { + const startIndex = workerNumber * pagesPerThread const pageData = this.context.pages.slice( startIndex, startIndex + pagesPerThread - ); + ) const pages = pageData.map(p => ({ path: p.path, frontmatter: JSON.stringify(p.frontmatter) - })); + })) const payload = { clientManifest: JSON.stringify(clientManifest), @@ -118,63 +115,63 @@ module.exports = class Build extends EventEmitter { serverBundle: JSON.stringify(serverBundle), siteConfig: JSON.stringify(this.context.siteConfig), ssrTemplate: JSON.stringify(this.context.ssrTemplate), - verbose: verbose, - workerNumber: i - }; + workerNumber, + logLevel: logger.options.logLevel + } - const worker = new Worker(path.join(__dirname, "./worker.js")); - worker.postMessage(payload); - activeWorkers++; - worker.on("message", response => { + const worker = new Worker(path.join(__dirname, './worker.js')) + worker.postMessage(payload) + activeWorkers++ + worker.on('message', response => { if (response.complete) { - pagePaths.concat(response.filePaths); + pagePaths.concat(response.filePaths) } if (response.message) { - logger.wait(response.message); + logger.wait(response.message) } - }); - worker.on("error", error => { + }) + worker.on('error', error => { // readline.cursorTo(process.stdout, 0, i) // readline.clearLine(process.stdout, 0) console.error( logger.error( - chalk.red(`Worker #${i} sent error: ${error}\n\n${error.stack}`), + chalk.red(`Worker #${workerNumber} sent error: ${error}\n\n${error.stack}`), false ) - ); - }); - worker.on("exit", code => { - activeWorkers--; + ) + }) + worker.on('exit', code => { + activeWorkers-- // readline.cursorTo(process.stdout, 0, i) // readline.clearLine(process.stdout, 0) if (code === 0) { - logger.success(`Worker ${i} completed successfully.`); + logger.success(`Worker ${workerNumber} completed successfully.`) } else { logger.error( - chalk.red(`Worker #${i} sent exit code: ${code}`), + chalk.red(`Worker #${workerNumber} sent exit code: ${code}`), false - ); + ) } if (activeWorkers === 0) { // DONE. - readline.clearLine(process.stdout, 0); - readline.cursorTo(process.stdout, 0); - const relativeDir = path.relative(this.context.cwd, this.outDir); + readline.clearLine(process.stdout, 0) + readline.cursorTo(process.stdout, 0) + const relativeDir = path.relative(this.context.cwd, this.outDir) logger.success( `Generated static files in ${chalk.cyan(relativeDir)}.` - ); - const { duration } = performance.stop(); + ) + const { duration } = performance.stop() logger.success( `It took a total of ${chalk.cyan( `${duration}ms` - )} to run the ${chalk.cyan("vuepress build")}.` - ); - console.log(); + )} to run the ${chalk.cyan('vuepress build')}.` + ) + console.log() } - }); + }) } - await this.context.pluginAPI.applyAsyncOption("generated", pagePaths); + await this.context.pluginAPI.applyAsyncOption('generated', pagePaths) } /** @@ -183,25 +180,25 @@ module.exports = class Build extends EventEmitter { * @api private */ - prepareWebpackConfig() { - this.clientConfig = createClientConfig(this.context).toConfig(); - this.serverConfig = createServerConfig(this.context).toConfig(); + prepareWebpackConfig () { + this.clientConfig = createClientConfig(this.context).toConfig() + this.serverConfig = createServerConfig(this.context).toConfig() - const userConfig = this.context.siteConfig.configureWebpack; + const userConfig = this.context.siteConfig.configureWebpack if (userConfig) { this.clientConfig = applyUserWebpackConfig( userConfig, this.clientConfig, false - ); + ) this.serverConfig = applyUserWebpackConfig( userConfig, this.serverConfig, true - ); + ) } } -}; +} /** * Compile a webpack application and return stats json. @@ -210,27 +207,27 @@ module.exports = class Build extends EventEmitter { * @returns {Promise} */ -function compile(config) { +function compile (config) { return new Promise((resolve, reject) => { webpack(config, (err, stats) => { if (err) { - return reject(err); + return reject(err) } if (stats.hasErrors()) { stats.toJson().errors.forEach(err => { - console.error(err); - }); - reject(new Error(`Failed to compile with errors.`)); - return; + console.error(err) + }) + reject(new Error(`Failed to compile with errors.`)) + return } if (env.isDebug && stats.hasWarnings()) { stats.toJson().warnings.forEach(warning => { - console.warn(warning); - }); + console.warn(warning) + }) } - resolve(stats.toJson({ modules: false })); - }); - }); + resolve(stats.toJson({ modules: false })) + }) + }) } /** @@ -243,20 +240,20 @@ function compile(config) { * @returns {Promise} */ -async function workaroundEmptyStyleChunk(stats, outDir) { +async function workaroundEmptyStyleChunk (stats, outDir) { const styleChunk = stats.children[0].assets.find(a => { - return /styles\.\w{8}\.js$/.test(a.name); - }); - if (!styleChunk) return; - const styleChunkPath = path.resolve(outDir, styleChunk.name); - const styleChunkContent = await fs.readFile(styleChunkPath, "utf-8"); - await fs.remove(styleChunkPath); + return /styles\.\w{8}\.js$/.test(a.name) + }) + if (!styleChunk) return + const styleChunkPath = path.resolve(outDir, styleChunk.name) + const styleChunkContent = await fs.readFile(styleChunkPath, 'utf-8') + await fs.remove(styleChunkPath) // prepend it to app.js. // this is necessary for the webpack runtime to work properly. const appChunk = stats.children[0].assets.find(a => { - return /app\.\w{8}\.js$/.test(a.name); - }); - const appChunkPath = path.resolve(outDir, appChunk.name); - const appChunkContent = await fs.readFile(appChunkPath, "utf-8"); - await fs.writeFile(appChunkPath, styleChunkContent + appChunkContent); + return /app\.\w{8}\.js$/.test(a.name) + }) + const appChunkPath = path.resolve(outDir, appChunk.name) + const appChunkContent = await fs.readFile(appChunkPath, 'utf-8') + await fs.writeFile(appChunkPath, styleChunkContent + appChunkContent) } diff --git a/packages/@vuepress/core/lib/node/build/worker.js b/packages/@vuepress/core/lib/node/build/worker.js index c5f184efaa..d7edc231bc 100644 --- a/packages/@vuepress/core/lib/node/build/worker.js +++ b/packages/@vuepress/core/lib/node/build/worker.js @@ -1,10 +1,9 @@ -const { parentPort } = require("worker_threads"); -const escape = require("escape-html"); -const readline = require("readline"); -const { chalk, fs, path, logger } = require("@vuepress/shared-utils"); -const { createBundleRenderer } = require("vue-server-renderer"); -const { normalizeHeadTag } = require("../util/index"); -const { version } = require("../../../package"); +const { parentPort } = require('worker_threads') +const escape = require('escape-html') +const { chalk, fs, path, logger } = require('@vuepress/shared-utils') +const { createBundleRenderer } = require('vue-server-renderer') +const { normalizeHeadTag } = require('../util/index') +const { version } = require('../../../package') /** * Worker file for HTML page rendering @@ -15,9 +14,10 @@ const { version } = require("../../../package"); * @api private */ -parentPort.once("message", async payload => { - const siteConfig = JSON.parse(payload.siteConfig); - const ssrTemplate = JSON.parse(payload.ssrTemplate); +parentPort.once('message', async payload => { + logger.setOptions({ logLevel: payload.logLevel }) + const siteConfig = JSON.parse(payload.siteConfig) + const ssrTemplate = JSON.parse(payload.ssrTemplate) // create server renderer using built manifests const renderer = createBundleRenderer(JSON.parse(payload.serverBundle), { @@ -25,53 +25,40 @@ parentPort.once("message", async payload => { runInNewContext: false, inject: false, shouldPrefetch: siteConfig.shouldPrefetch || (() => true), - template: await fs.readFile(ssrTemplate, "utf-8") - }); + template: await fs.readFile(ssrTemplate, 'utf-8') + }) // pre-render head tags from user config - const userHeadTags = (siteConfig.head || []).map(renderHeadTag).join("\n "); + const userHeadTags = (siteConfig.head || []).map(renderHeadTag).join('\n ') - const pages = JSON.parse(Buffer.from(payload.pages)); - // readline.cursorTo(process.stdout, 0, payload.workerNumber) - // readline.clearLine(process.stdout, 0) - process.stdout.write( - `Worker #${payload.workerNumber} beginning rendering of ${pages.length} pages\n` - ); - const filePaths = []; - let pagesRendered = 0; + const pages = JSON.parse(Buffer.from(payload.pages)) + logger.wait(`Worker #${payload.workerNumber} beginning rendering of ${pages.length} pages`) + const filePaths = [] + let pagesRendered = 0 for (const page of pages) { - const pagePath = decodeURIComponent(page.path); - if (payload.verbose) { - // readline.cursorTo(process.stdout, 0, payload.workerNumber) - // readline.clearLine(process.stdout, 0) - process.stdout.write( - `Worker #${payload.workerNumber} rendering page: ${pagePath}\n` - ); - } + const pagePath = decodeURIComponent(page.path) // #565 Avoid duplicate description meta at SSR. const meta = ((page.frontmatter && page.frontmatter.meta) || []).filter( - item => item.name !== "description" - ); - const pageMeta = renderPageMeta(meta); + item => item.name !== 'description' + ) + const pageMeta = renderPageMeta(meta) const context = { url: page.path, userHeadTags: userHeadTags, pageMeta, - title: "VuePress", - lang: "en", - description: "", + title: 'VuePress', + lang: 'en', + description: '', version - }; + } - let html; + let html try { - html = await renderer.renderToString(context); + html = await renderer.renderToString(context) } catch (e) { - // readline.cursorTo(process.stdout, 0, payload.workerNumber) - // readline.clearLine(process.stdout, 0) console.error( logger.error( chalk.red( @@ -79,24 +66,24 @@ parentPort.once("message", async payload => { ), false ) - ); - throw e; + ) + throw e } finally { const filename = pagePath - .replace(/\/$/, "/index.html") - .replace(/^\//, ""); - const filePath = path.resolve(payload.outDir, filename); - await fs.ensureDir(path.dirname(filePath)); - await fs.writeFile(filePath, html); - filePaths.push(filePath); - pagesRendered++; + .replace(/\/$/, '/index.html') + .replace(/^\//, '') + const filePath = path.resolve(payload.outDir, filename) + await fs.ensureDir(path.dirname(filePath)) + await fs.writeFile(filePath, html) + filePaths.push(filePath) + pagesRendered++ if (pagesRendered % 50 === 0) { parentPort.postMessage({ complete: false, message: `Worker #${payload.workerNumber} has rendered ${pagesRendered} of ${pages.length} pages`, filePaths: null - }); + }) } } } @@ -104,8 +91,8 @@ parentPort.once("message", async payload => { complete: true, message: `Worker #${payload.workerNumber} has rendered ${pagesRendered} of ${pages.length} pages`, filePaths: filePaths - }); -}); + }) +}) /** * Render html attributes @@ -114,12 +101,12 @@ parentPort.once("message", async payload => { * @returns {string} */ -function renderAttrs(attrs = {}) { - const keys = Object.keys(attrs); +function renderAttrs (attrs = {}) { + const keys = Object.keys(attrs) if (keys.length) { - return " " + keys.map(name => `${name}="${escape(attrs[name])}"`).join(" "); + return ' ' + keys.map(name => `${name}="${escape(attrs[name])}"`).join(' ') } else { - return ""; + return '' } } @@ -130,11 +117,11 @@ function renderAttrs(attrs = {}) { * @returns {string} */ -function renderHeadTag(tag) { - const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag); +function renderHeadTag (tag) { + const { tagName, attributes, innerHTML, closeTag } = normalizeHeadTag(tag) return `<${tagName}${renderAttrs(attributes)}>${innerHTML}${ closeTag ? `` : `` - }`; + }` } /** @@ -144,15 +131,15 @@ function renderHeadTag(tag) { * @returns {Array} */ -function renderPageMeta(meta) { - if (!meta) return ""; +function renderPageMeta (meta) { + if (!meta) return '' return meta .map(m => { - let res = ` { - res += ` ${key}="${escape(m[key])}"`; - }); - return res + `>`; + res += ` ${key}="${escape(m[key])}"` + }) + return res + `>` }) - .join(""); + .join('') } diff --git a/packages/docs/docs/api/cli.md b/packages/docs/docs/api/cli.md index b28b783dfc..2df6fbb1a4 100644 --- a/packages/docs/docs/api/cli.md +++ b/packages/docs/docs/api/cli.md @@ -16,6 +16,9 @@ See [port](../config/README.md#port). ### -t, --temp `` See [temp](../config/README.md#temp). +### -w, --worker `<#>` +Specifies the number of Node [worker_threads](https://nodejs.org/api/worker_threads.html) to use. Defaults to `1`. + ### -c, --cache `[cache]` ### --no-cache See [cache](../config/README.md#cache). diff --git a/packages/vuepress/lib/registerCoreCommands.js b/packages/vuepress/lib/registerCoreCommands.js index 2066565e97..62ce053270 100644 --- a/packages/vuepress/lib/registerCoreCommands.js +++ b/packages/vuepress/lib/registerCoreCommands.js @@ -47,17 +47,18 @@ module.exports = function (cli, options) { .option('-d, --dest ', 'specify build output dir (default: .vuepress/dist)') .option('-t, --temp ', 'set the directory of the temporary file') .option('-c, --cache [cache]', 'set the directory of cache') + .option('-w, --workers <#>', 'set the number of worker threads') .option('--dest ', 'the output directory for build process') .option('--no-cache', 'clean the cache before build') .option('--debug', 'build in development mode for debugging') .option('--silent', 'build static site in silent mode') .action((sourceDir = '.', commandOptions) => { - const { debug, silent } = commandOptions + const { debug, silent, workers } = commandOptions logger.setOptions({ logLevel: silent ? 1 : debug ? 4 : 3 }) logger.debug('global_options', options) logger.debug('build_options', commandOptions) - env.setOptions({ isDebug: debug, isTest: process.env.NODE_ENV === 'test' }) + env.setOptions({ isDebug: debug, isTest: process.env.NODE_ENV === 'test', workerThreads: workers || 1 }) wrapCommand(build)({ sourceDir: path.resolve(sourceDir), From 8166089557777a337b50a456e5e502e44d676b88 Mon Sep 17 00:00:00 2001 From: Mike Wright Date: Mon, 17 Feb 2020 15:33:06 -0700 Subject: [PATCH 4/4] docs: change Node to Node.js in cli docs --- packages/docs/docs/api/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docs/docs/api/cli.md b/packages/docs/docs/api/cli.md index 2df6fbb1a4..417963494d 100644 --- a/packages/docs/docs/api/cli.md +++ b/packages/docs/docs/api/cli.md @@ -17,7 +17,7 @@ See [port](../config/README.md#port). See [temp](../config/README.md#temp). ### -w, --worker `<#>` -Specifies the number of Node [worker_threads](https://nodejs.org/api/worker_threads.html) to use. Defaults to `1`. +Specifies the number of Node.js [worker_threads](https://nodejs.org/api/worker_threads.html) to use. Defaults to `1`. ### -c, --cache `[cache]` ### --no-cache