test: fix fetch-handler test flakiness (#170)

* test: ensure blobServer and test apiServer are not overlapping

* test: allow shards to continue running when one of shards fail

---------

Co-authored-by: Matt Kane <m@mk.gg>

Author: pieh

.github/workflows/run-tests.yml

@@ -57,6 +57,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         shard: [1/5, 2/5, 3/5, 4/5, 5/5]
     steps:

tests/integration/fetch-handler.test.ts

@@ -3,7 +3,7 @@ import getPort from 'get-port'
 import { getLogger } from 'lambda-local'
 import { createServer, type Server } from 'node:http'
 import { v4 } from 'uuid'
-import { afterAll, beforeAll, beforeEach, expect, test, vi } from 'vitest'
+import { afterEach, beforeEach, expect, test, vi } from 'vitest'
 import {
   createFixture,
   invokeFunction,
@@ -22,6 +22,10 @@ import {
 // Disable the verbose logging of the lambda-local runtime
 getLogger().level = 'alert'
 
+let apiBase: string
+let testServer: Server
+let handlerCalled = 0
+
 beforeEach<FixtureTestContext>(async (ctx) => {
   // set for each test a new deployID and siteID
   ctx.deployID = generateRandomObjectID()
@@ -32,15 +36,10 @@ beforeEach<FixtureTestContext>(async (ctx) => {
   // vi.spyOn(console, 'debug').mockImplementation(() => {})
 
   await startMockBlobStore(ctx)
-})
-
-let apiBase: string
-let testServer: Server
-let handlerCalled = 0
 
-beforeAll(async () => {
   // create a fake endpoint to test if it got called
-  const port = await getPort({ host: '0.0.0.0' })
+  const port = await getPort({ host: '0.0.0.0', exclude: [ctx.blobStorePort] })
+  handlerCalled = 0
 
   testServer = createServer((_, res) => {
     handlerCalled++
@@ -56,7 +55,7 @@ beforeAll(async () => {
   })
 })
 
-afterAll(async () => {
+afterEach(async () => {
   testServer.closeAllConnections()
   await new Promise((resolve) => {
     testServer.on('close', resolve)

tests/utils/fixture.ts

@@ -30,6 +30,7 @@ export interface FixtureTestContext extends TestContext {
   siteID: string
   deployID: string
   blobStoreHost: string
+  blobStorePort: number
   blobServer: BlobsServer
   blobStore: ReturnType<typeof getStore>
   functionDist: string

tests/utils/helpers.ts

@@ -63,6 +63,7 @@ export const startMockBlobStore = async (ctx: FixtureTestContext) => {
   })
   await ctx.blobServer.start()
   ctx.blobStoreHost = `localhost:${port}`
+  ctx.blobStorePort = port
   vi.stubEnv('NETLIFY_BLOBS_CONTEXT', createBlobContext(ctx))
 
   ctx.blobStore = getDeployStore({