perf: memoize blobs requests in the request scope

Author: pieh

package-lock.json

@@ -76,6 +76,7 @@
     "fs-monkey": "^1.0.6",
     "get-port": "^7.1.0",
     "lambda-local": "^2.2.0",
+    "lru-cache": "^10.4.3",
     "memfs": "^4.9.2",
     "mock-require": "^3.0.3",
     "msw": "^2.0.7",

package.json

@@ -30,22 +30,18 @@ import {
 import { getLogger, getRequestContext } from './request-context.cjs'
 import { getTracer, recordWarning } from './tracer.cjs'
 
-type TagManifestBlobCache = Record<string, Promise<TagManifest | null>>
-
 const purgeCacheUserAgent = `${nextRuntimePkgName}@${nextRuntimePkgVersion}`
 
 export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
   options: CacheHandlerContext
   revalidatedTags: string[]
   cacheStore: MemoizedKeyValueStoreBackedByRegionalBlobStore
   tracer = getTracer()
-  tagManifestsFetchedFromBlobStoreInCurrentRequest: TagManifestBlobCache
 
   constructor(options: CacheHandlerContext) {
     this.options = options
     this.revalidatedTags = options.revalidatedTags
     this.cacheStore = getMemoizedKeyValueStoreBackedByRegionalBlobStore({ consistency: 'strong' })
-    this.tagManifestsFetchedFromBlobStoreInCurrentRequest = {}
   }
 
   private getTTL(blob: NetlifyCacheHandlerValue) {
@@ -469,7 +465,8 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
   }
 
   resetRequestCache() {
-    this.tagManifestsFetchedFromBlobStoreInCurrentRequest = {}
+    // no-op because in-memory cache is scoped to requests and not global
+    // see getRequestSpecificInMemoryCache
   }
 
   /**
@@ -508,10 +505,9 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
     }
 
     // 2. If any in-memory tags don't indicate that any of tags was invalidated
-    //    we will check blob store, but memoize results for duration of current request
-    //    so that we only check blob store once per tag within a single request
-    //    full-route cache and fetch caches share a lot of tags so this might save
-    //    some roundtrips to the blob store.
+    //    we will check blob store. Full-route cache and fetch caches share a lot of tags
+    //    but we will only do actual blob read once withing a single request due to cacheStore
+    //    memoization.
     //    Additionally, we will resolve the promise as soon as we find first
     //    stale tag, so that we don't wait for all of them to resolve (but keep all
     //    running in case future `CacheHandler.get` calls would be able to use results).
@@ -521,14 +517,10 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
       const tagManifestPromises: Promise<boolean>[] = []
 
       for (const tag of cacheTags) {
-        let tagManifestPromise: Promise<TagManifest | null> =
-          this.tagManifestsFetchedFromBlobStoreInCurrentRequest[tag]
-
-        if (!tagManifestPromise) {
-          tagManifestPromise = this.cacheStore.get<TagManifest>(tag, 'tagManifest.get')
-
-          this.tagManifestsFetchedFromBlobStoreInCurrentRequest[tag] = tagManifestPromise
-        }
+        const tagManifestPromise: Promise<TagManifest | null> = this.cacheStore.get<TagManifest>(
+          tag,
+          'tagManifest.get',
+        )
 
         tagManifestPromises.push(
           tagManifestPromise.then((tagManifest) => {

src/run/handlers/cache.cts

@@ -39,9 +39,22 @@ export type RequestContext = {
    */
   backgroundWorkPromise: Promise<unknown>
   logger: SystemLogger
+  requestID: string
 }
 
 type RequestContextAsyncLocalStorage = AsyncLocalStorage<RequestContext>
+const REQUEST_CONTEXT_GLOBAL_KEY = Symbol.for('nf-request-context-async-local-storage')
+const REQUEST_COUNTER_KEY = Symbol.for('nf-request-counter')
+const extendedGlobalThis = globalThis as typeof globalThis & {
+  [REQUEST_CONTEXT_GLOBAL_KEY]?: RequestContextAsyncLocalStorage
+  [REQUEST_COUNTER_KEY]?: number
+}
+
+function getFallbackRequestID() {
+  const requestNumber = extendedGlobalThis[REQUEST_COUNTER_KEY] ?? 0
+  extendedGlobalThis[REQUEST_COUNTER_KEY] = requestNumber + 1
+  return `#${requestNumber}`
+}
 
 export function createRequestContext(request?: Request, context?: FutureContext): RequestContext {
   const backgroundWorkPromises: Promise<unknown>[] = []
@@ -72,10 +85,10 @@ export function createRequestContext(request?: Request, context?: FutureContext)
       return Promise.allSettled(backgroundWorkPromises)
     },
     logger,
+    requestID: request?.headers.get('x-nf-request-id') ?? getFallbackRequestID(),
   }
 }
 
-const REQUEST_CONTEXT_GLOBAL_KEY = Symbol.for('nf-request-context-async-local-storage')
 let requestContextAsyncLocalStorage: RequestContextAsyncLocalStorage | undefined
 function getRequestContextAsyncLocalStorage() {
   if (requestContextAsyncLocalStorage) {
@@ -85,9 +98,6 @@ function getRequestContextAsyncLocalStorage() {
   // AsyncLocalStorage in the module scope, because it will be different for each
   // copy - so first time an instance of this module is used, we store AsyncLocalStorage
   // in global scope and reuse it for all subsequent calls
-  const extendedGlobalThis = globalThis as typeof globalThis & {
-    [REQUEST_CONTEXT_GLOBAL_KEY]?: RequestContextAsyncLocalStorage
-  }
   if (extendedGlobalThis[REQUEST_CONTEXT_GLOBAL_KEY]) {
     return extendedGlobalThis[REQUEST_CONTEXT_GLOBAL_KEY]
   }

src/run/handlers/request-context.cts

@@ -1,7 +1,9 @@
 import { getDeployStore, GetWithMetadataOptions, Store } from '@netlify/blobs'
+import { LRUCache } from 'lru-cache'
 
-import type { BlobType } from '../shared/cache-types.cjs'
+import { type BlobType, estimateBlobSize } from '../shared/cache-types.cjs'
 
+import { getRequestContext } from './handlers/request-context.cjs'
 import { getTracer } from './handlers/tracer.cjs'
 
 const FETCH_BEFORE_NEXT_PATCHED_IT = Symbol.for('nf-not-patched-fetch')
@@ -70,6 +72,54 @@ const encodeBlobKey = async (key: string) => {
   return await encodeBlobKeyImpl(key)
 }
 
+// lru-cache types don't like using `null` for values, so we use a symbol to represent it and do conversion
+// so it doesn't leak outside
+const NullValue = Symbol.for('null-value')
+const inMemoryLRUCache = new LRUCache<
+  string,
+  BlobType | typeof NullValue | Promise<BlobType | null>
+>({
+  max: 1000,
+  // TODO: get value from CacheHandler configuration
+  maxEntrySize: 50 * 1024 * 1024, // 50MB
+  sizeCalculation: (valueToStore) => {
+    return estimateBlobSize(valueToStore === NullValue ? null : valueToStore)
+  },
+})
+
+interface RequestSpecificInMemoryCache {
+  get(key: string): BlobType | null | Promise<BlobType | null> | undefined
+  set(key: string, value: BlobType | null | Promise<BlobType | null>): void
+}
+
+const getRequestSpecificInMemoryCache = (): RequestSpecificInMemoryCache => {
+  const requestContext = getRequestContext()
+  if (!requestContext) {
+    // Fallback to a no-op store if we can't find request context
+    return {
+      get(): undefined {
+        // no-op
+      },
+      set() {
+        // no-op
+      },
+    }
+  }
+
+  return {
+    get(key) {
+      const inMemoryValue = inMemoryLRUCache.get(`${requestContext.requestID}:${key}`)
+      if (inMemoryValue === NullValue) {
+        return null
+      }
+      return inMemoryValue
+    },
+    set(key, value) {
+      inMemoryLRUCache.set(`${requestContext.requestID}:${key}`, value ?? NullValue)
+    },
+  }
+}
+
 export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
   args: GetWithMetadataOptions = {},
 ) => {
@@ -78,18 +128,30 @@ export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
 
   return {
     async get<T extends BlobType>(key: string, otelSpanTitle: string): Promise<T | null> {
-      const blobKey = await encodeBlobKey(key)
+      const inMemoryCache = getRequestSpecificInMemoryCache()
 
-      return tracer.withActiveSpan(otelSpanTitle, async (span) => {
+      const memoizedValue = inMemoryCache.get(key)
+      if (typeof memoizedValue !== 'undefined') {
+        return memoizedValue as T | null | Promise<T | null>
+      }
+
+      const blobKey = await encodeBlobKey(key)
+      const getPromise = tracer.withActiveSpan(otelSpanTitle, async (span) => {
         span.setAttributes({ key, blobKey })
         const blob = (await store.get(blobKey, { type: 'json' })) as T | null
+        inMemoryCache.set(key, blob)
         span.addEvent(blob ? 'Hit' : 'Miss')
         return blob
       })
+      inMemoryCache.set(key, getPromise)
+      return getPromise
     },
     async set(key: string, value: BlobType, otelSpanTitle: string) {
-      const blobKey = await encodeBlobKey(key)
+      const inMemoryCache = getRequestSpecificInMemoryCache()
 
+      inMemoryCache.set(key, value)
+
+      const blobKey = await encodeBlobKey(key)
       return tracer.withActiveSpan(otelSpanTitle, async (span) => {
         span.setAttributes({ key, blobKey })
         return await store.setJSON(blobKey, value)