chore: map gh issues to failures (#532)

* gh issues to failure list

* add token

* add based on file name or test name

* add comments

* Update tools/deno/ghIssues2json.ts

Co-authored-by: Philippe Serhal <philippe.serhal@netlify.com>

* return empty arr if no match

* allow for multi test names

* add schedule

* fix multi name setup

* Update tools/deno/ghIssues2json.ts

Co-authored-by: Philippe Serhal <philippe.serhal@netlify.com>

* Update tools/deno/ghIssues2json.ts

Co-authored-by: Philippe Serhal <philippe.serhal@netlify.com>

* chore: format with prettier

---------

Co-authored-by: Philippe Serhal <philippe.serhal@netlify.com>
Co-authored-by: taty2010 <taty2010@users.noreply.github.com>

Author: taty2010

.github/workflows/e2e-report.yml

@@ -1,5 +1,7 @@
 name: 'Deploy e2e test page'
 on:
+  schedule:
+    - cron: '0 3 * * 3' # Run every Wednesday at 3am UTC
   workflow_dispatch:
     inputs:
       use-branch:

.github/workflows/test-e2e.yml

@@ -38,6 +38,7 @@ env:
   NEXT_TEST_CONTINUE_ON_ERROR: 1
   next-path: next.js
   runtime-path: next-runtime-minimal
+  GH_TOKEN: ${{ github.token }}
 jobs:
   setup:
     runs-on: ubuntu-latest
@@ -237,6 +238,12 @@ jobs:
           pattern: test-result-${{matrix.version_spec.selector}}-*
           path: artifacts
 
+      - name: Get and format Github issues
+        if: success() || failure()
+        run: |
+          gh issue list --label "Next.js e2e test failure" --json number,body > report/issues.json
+          deno run -A tools/deno/ghIssues2json.ts tests/test-config.json report/issues.json
+
       - name: Publish Test Report
         id: publish-test-results
         if: success() || failure()

tests/test-config.json

@@ -338,195 +338,5 @@
       "reason": "Test is incompatible with serverless because it relies on shared state between requests",
       "tests": ["should refresh the correct page when a server action triggers a redirect"]
     }
-  ],
-  "failures": [
-    {
-      "name": "Middleware Runtime without i18n should trigger middleware for data requests",
-      "reason": "Requests for page data on pages router sites with middleware return HTML unless x-nextjs-data header is set",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/380"
-    },
-    {
-      "name": "Middleware Runtime with i18n should trigger middleware for data requests",
-      "reason": "Requests for page data on pages router sites with middleware return HTML unless x-nextjs-data header is set",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/380"
-    },
-    {
-      "name": "app dir - basic bootstrap scripts should successfully bootstrap even when using CSP",
-      "reason": "Nonce not automatically set in script tags when using CSP",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/381"
-    },
-    {
-      "name": "app dir - basic next/script should pass `nonce`",
-      "reason": "Nonce not automatically set in script tags when using CSP",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/381"
-    },
-    {
-      "name": "app dir - basic next/script should pass nonce when using next/font",
-      "reason": "Nonce not automatically set in script tags when using CSP",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/381"
-    },
-    {
-      "name": "Middleware Runtime with i18n should validate & parse request url from a dynamic route with params",
-      "reason": "Middleware in sites with i18n generating incorrect request.url pathname",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/382"
-    },
-    {
-      "name": "Middleware Runtime with i18n should validate & parse request url from a dynamic route with params and no query",
-      "reason": "Middleware in sites with i18n generating incorrect request.url pathname",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/382"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale with basepath get public file by skipping locale in rewrite, locale: ",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale with basepath get public file by skipping locale in rewrite, locale: /en",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale with basepath get public file by skipping locale in rewrite, locale: /nl",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale with basepath get public file by skipping locale in rewrite, locale: /sv",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale get public file by skipping locale in rewrite, locale: ",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale get public file by skipping locale in rewrite, locale: /en",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale get public file by skipping locale in rewrite, locale: /nl",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "i18n-ignore-rewrite-source-locale get public file by skipping locale in rewrite, locale: /sv",
-      "reason": "Middleware on sites with i18n cannot rewrite to static files",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/383"
-    },
-    {
-      "name": "Middleware Runtime trailing slash should keep non data requests in their original shape",
-      "reason": "Middleware should not add trailing slashes to non-data requests in static dir",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/385"
-    },
-    {
-      "name": "Middleware Redirect should implement internal redirects",
-      "reason": "Pages router middleware should return 302 status for redirected data requests",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/386"
-    },
-    {
-      "name": "Middleware Redirect /fr should implement internal redirects",
-      "reason": "Pages router middleware should return 302 status for redirected data requests",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/386"
-    },
-    {
-      "name": "Middleware Redirect should redirect to api route with locale",
-      "reason": "Pages router middleware should return 302 status for redirected data requests",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/386"
-    },
-    {
-      "name": "Middleware Redirect /fr should redirect to api route with locale",
-      "reason": "Pages router middleware should return 302 status for redirected data requests",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/386"
-    },
-    {
-      "name": "getServerSideProps should handle throw ENOENT correctly",
-      "reason": "Server error pages return encoded data without content-encoding header if accept-encoding is gzip",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/387"
-    },
-    {
-      "name": "app-custom-routes works with generateStaticParams correctly responds correctly on /static/three/data.json",
-      "reason": "First request for generateStaticParams fallback route handler returns base64-encoded content",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/388"
-    },
-    {
-      "name": "app-custom-routes works with generateStaticParams correctly revalidates correctly on /revalidate-1/three/data.json",
-      "reason": "First request for generateStaticParams fallback route handler returns base64-encoded content",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/388"
-    },
-    {
-      "name": "app-dir action handling fetch actions should store revalidation data in the prefetch cache",
-      "reason": "Fetch action prefetch cache test is flakey",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/444"
-    },
-    {
-      "name": "parallel-routes-and-interception parallel routes should support parallel route tab bars",
-      "reason": "Some parallel routes returning 404",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/445"
-    },
-    {
-      "name": "parallel-routes-and-interception parallel routes should throw a 404 when no matching parallel route is found",
-      "reason": "Some parallel routes returning 404",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/445"
-    },
-    {
-      "name": "parallel-routes-and-interception parallel routes should only scroll to the parallel route that was navigated to",
-      "reason": "Some parallel routes returning 404",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/445"
-    },
-    {
-      "name": "Middleware Responses should respond appending headers headers",
-      "reason": "Appending set-cookie header in middleware leads to duplicate header",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/447"
-    },
-    {
-      "name": "Middleware Responses /fr should respond appending headers headers",
-      "reason": "Appending set-cookie header in middleware leads to duplicate header",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/447"
-    },
-    {
-      "name": "app-dir revalidate-dynamic should revalidate the data with /api/revalidate-path",
-      "reason": "Not a fail: race condition when testing revalidation"
-    },
-    {
-      "name": "app-dir revalidate-dynamic should revalidate the data with /api/revalidate-tag",
-      "reason": "Not a fail: race condition when testing revalidation"
-    },
-    {
-      "name": "Middleware base tests router.query must exist when Link clicked page routing",
-      "reason": "Pages router data requests returning 404 when middleware is used",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/450"
-    },
-    {
-      "name": "Middleware Rewrite should handle catch-all rewrite correctly",
-      "reason": "TODO: correct behaviour is unclear. See link for implementation",
-      "link": "https://github.com/vercel/next.js/pull/63254"
-    },
-    {
-      "name": "Middleware Rewrite should allow to rewrite keeping the locale in pathname",
-      "reason": "Middleware rewrites should keep the locale in the pathname",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/451"
-    },
-    {
-      "name": "parallel-routes-revalidation should submit the action and revalidate the page data",
-      "reason": "Parallel route revalidation actions are intermittently failing",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/452"
-    },
-    {
-      "name": "Middleware custom matchers i18n should not match",
-      "reason": "Middleware matching is too broad when using i18n",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/453"
-    },
-    {
-      "name": "using a single matcher with i18n and basePath adds the header for a matched path",
-      "reason": "Middleware does not match when using basePath and default locale",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/454"
-    },
-    {
-      "name": "using a single matcher with i18n and basePath and trailingSlash adds the header for a matched path",
-      "reason": "Middleware does not match when using basePath and default locale",
-      "link": "https://github.com/netlify/next-runtime-minimal/issues/454"
-    }
   ]
 }

tools/deno/ghIssues2json.ts

@@ -0,0 +1,37 @@
+interface Annotation {
+  link: string
+  name: string
+  reason: string
+}
+
+async function writeToConfig(
+  filePath: string,
+  failures: { link: string; name: string; reason: string }[],
+) {
+  const testConfig = JSON.parse(await Deno.readTextFile(filePath))
+  testConfig.failures = failures
+  await Deno.writeTextFile(filePath, JSON.stringify(testConfig))
+}
+
+async function formatIssues(file: string) {
+  const issues = JSON.parse(await Deno.readTextFile(file))
+  const annotations: Annotation[] = []
+
+  issues.forEach((issue: { body: string; number: number }) => {
+    const name = issue.body.match(/^test: (.+)$/m) || []
+    const reason = issue.body.match(/^reason: (.+)$/m) || []
+    const testNames = name[1]?.split(',')
+
+    testNames?.forEach((name) => {
+      annotations.push({
+        link: `https://github.com/netlify/next-runtime-minimal/issues/${issue.number}`,
+        reason: reason[1],
+        name: name.trim(),
+      })
+    })
+  })
+
+  writeToConfig(Deno.args[0], annotations)
+}
+
+formatIssues(Deno.args[1])

tools/deno/junit2json.ts

@@ -109,7 +109,9 @@ function junitToJson(xmlData: {
         status,
       }
       if (status === 'failed') {
-        const failure = testConfig.failures.find((conf) => conf.name === test.name)
+        const failure = testConfig.failures.find(
+          (conf) => conf.name === test.name || conf.name === testSuite.file,
+        )
         if (failure) {
           test.reason = failure.reason
           test.link = failure.link