fix: correct handling of CDN files

Author: ascorbic

demos/default/next.config.js

@@ -31,6 +31,10 @@ module.exports = {
           destination: '/:path*',
         },
       ],
+      afterFiles: [{
+        source: '/rewriteToStatic',
+        destination: '/getStaticProps/1',
+      }]
     }
   },
   // Redirects allow you to redirect an incoming request path to a different destination path.

demos/default/pages/index.js

@@ -167,6 +167,11 @@ const Index = ({ shows }) => {
             <a>Rewrite (should display image)</a>
           </Link>
         </li>
+        <li>
+          <Link href="/rewriteToStatic">
+            <a>Rewrite to static (should show getStaticProps/1)</a>
+          </Link>
+        </li>
         <li>
           <Link href="/middle">
             <a>Middleware</a>

src/helpers/files.js

@@ -2,7 +2,7 @@
 const { cpus } = require('os')
 
 const { yellowBright } = require('chalk')
-const { existsSync, readJson, move, cpSync, copy, writeJson, readFile, writeFile } = require('fs-extra')
+const { existsSync, readJson, move, copy, writeJson, readFile, writeFile, ensureDir } = require('fs-extra')
 const globby = require('globby')
 const { outdent } = require('outdent')
 const pLimit = require('p-limit')
@@ -62,7 +62,9 @@ exports.moveStaticPages = async ({ netlifyConfig, target, i18n }) => {
   console.log('Moving static page files to serve from CDN...')
   const outputDir = join(netlifyConfig.build.publish, target === 'server' ? 'server' : 'serverless')
   const root = join(outputDir, 'pages')
-
+  const buildId = await readFile(join(netlifyConfig.build.publish, 'BUILD_ID'), 'utf8')
+  const dataDir = join('_next', 'data', buildId)
+  await ensureDir(dataDir)
   // Load the middleware manifest so we can check if a file matches it before moving
   let middleware
   const manifestPath = join(outputDir, 'middleware-manifest.json')
@@ -88,10 +90,17 @@ exports.moveStaticPages = async ({ netlifyConfig, target, i18n }) => {
   })
 
   const files = []
+  const filesManifest = {}
   const moveFile = async (file) => {
+    const isData = file.endsWith('.json')
     const source = join(root, file)
+    const target = isData ? join(dataDir, file) : file
+
     files.push(file)
-    const dest = join(netlifyConfig.build.publish, file)
+    filesManifest[file] = target
+
+    const dest = join(netlifyConfig.build.publish, target)
+
     try {
       await move(source, dest)
     } catch (error) {
@@ -208,7 +217,7 @@ exports.moveStaticPages = async ({ netlifyConfig, target, i18n }) => {
   }
 
   // Write the manifest for use in the serverless functions
-  await writeJson(join(netlifyConfig.build.publish, 'static-manifest.json'), files)
+  await writeJson(join(netlifyConfig.build.publish, 'static-manifest.json'), Object.entries(filesManifest))
 
   if (i18n?.defaultLocale) {
     // Copy the default locale into the root

src/templates/getHandler.js

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines-per-function */
 const { promises, existsSync } = require('fs')
 const { Server } = require('http')
 const { tmpdir } = require('os')
@@ -34,12 +35,14 @@ const makeHandler =
       // In most cases these are served from the CDN, but for rewrites Next may try to read them
       // from disk. We need to intercept these and load them from the CDN instead
       // Sadly the only way to do this is to monkey-patch fs.promises. Yeah, I know.
-      const staticFiles = new Set(staticManifest)
-
+      const staticFiles = new Map(staticManifest)
+      const downloadPromises = new Map()
+      const statsCache = new Map()
       // Yes, you can cache stuff locally in a Lambda
       const cacheDir = path.join(tmpdir(), 'next-static-cache')
       // Grab the real fs.promises.readFile...
       const readfileOrig = promises.readFile
+      const statsOrig = promises.stat
       // ...then money-patch it to see if it's requesting a CDN file
       promises.readFile = async (file, options) => {
         // We only care about page files
@@ -51,13 +54,24 @@ const makeHandler =
           if (staticFiles.has(filePath) && !existsSync(file)) {
             // This name is safe to use, because it's one that was already created by Next
             const cacheFile = path.join(cacheDir, filePath)
-            // Have we already cached it? We ignore the cache if running locally to avoid staleness
+            const url = `${base}/${staticFiles.get(filePath)}`
+
+            // If it's already downloading we can wait for it to finish
+            if (downloadPromises.has(url)) {
+              await downloadPromises.get(url)
+            }
+            // Have we already cached it? We download every time if running locally to avoid staleness
             if ((!existsSync(cacheFile) || process.env.NETLIFY_DEV) && base) {
               await promises.mkdir(path.dirname(cacheFile), { recursive: true })
 
-              // Append the path to our host and we can load it like a regular page
-              const url = `${base}/${filePath}`
-              await downloadFile(url, cacheFile)
+              try {
+                // Append the path to our host and we can load it like a regular page
+                const downloadPromise = downloadFile(url, cacheFile)
+                downloadPromises.set(url, downloadPromise)
+                await downloadPromise
+              } finally {
+                downloadPromises.delete(url)
+              }
             }
             // Return the cache file
             return readfileOrig(cacheFile, options)
@@ -66,6 +80,18 @@ const makeHandler =
 
         return readfileOrig(file, options)
       }
+
+      promises.stat = async (file, options) => {
+        // We only care about page files
+        if (file.startsWith(pageRoot)) {
+          // We only want the part after `pages/`
+          const cacheFile = path.join(cacheDir, file.slice(pageRoot.length + 1))
+          if (existsSync(cacheFile)) {
+            return statsOrig(cacheFile, options)
+          }
+        }
+        return statsOrig(file, options)
+      }
     }
     let NextServer
     try {
@@ -183,3 +209,4 @@ exports.handler = ${
 `
 
 module.exports = getHandler
+/* eslint-enable max-lines-per-function */