Merge pull request #11674 from getsentry/develop

Gitflow: Sync develop into master

Author: mydea

.github/ISSUE_TEMPLATE/flaky.yml

@@ -18,7 +18,7 @@ body:
     id: job-name
     attributes:
       label: Name of Job
-      placeholder: Build & Test / Nextjs (Node 14) Tests
+      placeholder: "CI: Build & Test / Nextjs (Node 14) Tests"
       description: name of job as reported in the status report
     validations:
       required: true

.github/workflows/auto-release.yml

@@ -1,4 +1,4 @@
-name: Gitflow - Auto prepare release
+name: "Gitflow: Auto prepare release"
 on:
   pull_request:
     types:
@@ -25,7 +25,7 @@ jobs:
           # Parse version from head branch
           text: ${{ github.head_ref }}
           # match: preprare-release/xx.xx.xx
-          regex: '^prepare-release\/(\d+\.\d+\.\d+)$'
+          regex: '^prepare-release\/(\d+\.\d+\.\d+)(?:-(alpha|beta)\.\d+)?$'
 
       - name: Prepare release
         uses: getsentry/action-prepare-release@v1

.github/workflows/build.yml

@@ -1,11 +1,13 @@
-name: 'Build & Test'
+name: 'CI: Build & Test'
 on:
   push:
     branches:
       - develop
       - master
       - release/**
   pull_request:
+  merge_group:
+    types: [checks_requested]
   workflow_dispatch:
     inputs:
       commit:
@@ -1051,6 +1053,7 @@ jobs:
             'node-nestjs-app',
             'node-exports-test-app',
             'node-koa-app',
+            'node-connect-app',
             'vue-3',
             'webpack-4',
             'webpack-5'

.github/workflows/canary.yml

@@ -1,4 +1,4 @@
-name: 'Canary Tests'
+name: 'CI: Canary Tests'
 on:
   schedule:
     # Run every day at midnight

.github/workflows/clear-cache.yml

@@ -1,4 +1,4 @@
-name: Clear all GHA caches
+name: "Action: Clear all GHA caches"
 on:
   workflow_dispatch:
 

.github/workflows/codeql-analysis.yml

@@ -9,7 +9,7 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: 'CodeQL'
+name: 'CI: CodeQL'
 
 on:
   push:

.github/workflows/enforce-license-compliance.yml

@@ -1,4 +1,4 @@
-name: Enforce License Compliance
+name: "CI: Enforce License Compliance"
 
 on:
   push:

.github/workflows/flaky-test-detector.yml

@@ -1,4 +1,4 @@
-name: 'Detect flaky tests'
+name: 'CI: Detect flaky tests'
 on:
   workflow_dispatch:
   pull_request:

.github/workflows/gitflow-sync-develop.yml

@@ -1,4 +1,4 @@
-name: Gitflow - Sync master into develop
+name: "Gitflow: Sync master into develop"
 on:
   push:
     branches:

.github/workflows/issue-package-label.yml

@@ -1,4 +1,4 @@
-name: 'Tag issue with package label'
+name: 'Automation: Tag issue with package label'
 
 on:
   issues:

.github/workflows/release-size-info.yml

@@ -1,4 +1,4 @@
-name: Add size info to release
+name: "Automation: Add size info to release"
 on:
   release:
     types:
@@ -27,4 +27,4 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           version: ${{ steps.get_version.outputs.version }}
-          workflow_name: 'Build & Test'
+          workflow_name: 'CI: Build & Test'

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Prepare Release
+name: "Action: Prepare Release"
 on:
   workflow_dispatch:
     inputs:

README.md

@@ -8,8 +8,8 @@ _Bad software is everywhere, and we're tired of it. Sentry is on a mission to he
 faster, so we can get back to enjoying technology. If you want to join us
 [<kbd>**Check out our open positions**</kbd>](https://sentry.io/careers/)_
 
-![Build & Test](https://github.com/getsentry/sentry-javascript/workflows/Build%20&%20Test/badge.svg)
-[![codecov](https://codecov.io/gh/getsentry/sentry-javascript/branch/master/graph/badge.svg)](https://codecov.io/gh/getsentry/sentry-javascript)
+![Build & Test](https://github.com/getsentry/sentry-javascript/workflows/CI:%20Build%20&%20Test/badge.svg)
+[![codecov](https://codecov.io/gh/getsentry/sentry-javascript/branch/develop/graph/badge.svg)](https://codecov.io/gh/getsentry/sentry-javascript)
 [![npm version](https://img.shields.io/npm/v/@sentry/core.svg)](https://www.npmjs.com/package/@sentry/core)
 [![Discord](https://img.shields.io/discord/621778831602221064)](https://discord.gg/Ww9hbqr)
 

dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts

@@ -1,7 +1,30 @@
 import * as childProcess from 'child_process';
+import * as fs from 'fs';
 import * as path from 'path';
 import * as glob from 'glob';
 
+/**
+ * The number of browsers we run the tests in.
+ */
+const NUM_BROWSERS = 3;
+
+/**
+ * Assume that each test runs for 3s.
+ */
+const ASSUMED_TEST_DURATION_SECONDS = 3;
+
+/**
+ * We keep the runtime of the detector if possible under 30min.
+ */
+const MAX_TARGET_TEST_RUNTIME_SECONDS = 30 * 60;
+
+/**
+ * Running one test 50x is what we consider enough to detect flakiness.
+ * Running one test 5x is the bare minimum
+ */
+const MAX_PER_TEST_RUN_COUNT = 50;
+const MIN_PER_TEST_RUN_COUNT = 5;
+
 async function run(): Promise<void> {
   let testPaths: string[] = [];
 
@@ -20,23 +43,8 @@ ${changedPaths.join('\n')}
     }
   }
 
-  let runCount: number;
-  if (process.env.TEST_RUN_COUNT === 'AUTO') {
-    // No test paths detected: run everything 5x
-    runCount = 5;
-
-    if (testPaths.length > 0) {
-      // Run everything up to 100x, assuming that total runtime is less than 60min.
-      // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
-      // We want to keep overall runtime under 30min
-      const testCount = testPaths.length * 4;
-      const expectedRuntimePerTestPath = testCount * 3;
-      const expectedRuntime = Math.floor((30 * 60) / expectedRuntimePerTestPath);
-      runCount = Math.min(50, Math.max(expectedRuntime, 5));
-    }
-  } else {
-    runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
-  }
+  const repeatEachCount = getPerTestRunCount(testPaths);
+  console.log(`Running tests ${repeatEachCount} times each.`);
 
   const cwd = path.join(__dirname, '../');
 
@@ -45,7 +53,7 @@ ${changedPaths.join('\n')}
       const cp = childProcess.spawn(
         `npx playwright test ${
           testPaths.length ? testPaths.join(' ') : './suites'
-        } --reporter='line' --repeat-each ${runCount}`,
+        } --reporter='line' --repeat-each ${repeatEachCount}`,
         { shell: true, cwd },
       );
 
@@ -88,6 +96,33 @@ ${changedPaths.join('\n')}
   console.log(`☑️ All tests passed.`);
 }
 
+/**
+ * Returns how many time one test should run based on the chosen mode and a bunch of heuristics
+ */
+function getPerTestRunCount(testPaths: string[]) {
+  if (process.env.TEST_RUN_COUNT === 'AUTO' && testPaths.length > 0) {
+    // Run everything up to 100x, assuming that total runtime is less than 60min.
+    // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
+    // We want to keep overall runtime under 30min
+    const estimatedNumberOfTests = testPaths.map(getApproximateNumberOfTests).reduce((a, b) => a + b);
+    console.log(`Estimated number of tests: ${estimatedNumberOfTests}`);
+
+    // tests are usually run against all browsers we test with, so let's assume this
+    const testRunCount = estimatedNumberOfTests * NUM_BROWSERS;
+    console.log(`Estimated test runs for one round: ${testRunCount}`);
+
+    const estimatedTestRuntime = testRunCount * ASSUMED_TEST_DURATION_SECONDS;
+    console.log(`Estimated test runtime: ${estimatedTestRuntime}s`);
+
+    const expectedPerTestRunCount = Math.floor(MAX_TARGET_TEST_RUNTIME_SECONDS / estimatedTestRuntime);
+    console.log(`Expected per-test run count: ${expectedPerTestRunCount}`);
+
+    return Math.min(MAX_PER_TEST_RUN_COUNT, Math.max(expectedPerTestRunCount, MIN_PER_TEST_RUN_COUNT));
+  }
+
+  return parseInt(process.env.TEST_RUN_COUNT || '5');
+}
+
 function getTestPaths(): string[] {
   const paths = glob.sync('suites/**/test.{ts,js}', {
     cwd: path.join(__dirname, '../'),
@@ -111,4 +146,22 @@ function logError(error: unknown) {
   }
 }
 
+/**
+ * Definitely not bulletproof way of getting the number of tests in a file :D
+ * We simply match on `it(`, `test(`, etc and count the matches.
+ *
+ * Note: This test completely disregards parameterized tests (`it.each`, etc) or
+ * skipped/disabled tests and other edge cases. It's just a rough estimate.
+ */
+function getApproximateNumberOfTests(testPath: string): number {
+  try {
+    const content = fs.readFileSync(path.join(process.cwd(), testPath, 'test.ts'), 'utf-8');
+    const matches = content.match(/it\(|test\(|sentryTest\(/g);
+    return Math.max(matches ? matches.length : 1, 1);
+  } catch (e) {
+    console.error(`Could not read file ${testPath}`);
+    return 1;
+  }
+}
+
 run();

dev-packages/e2e-tests/test-applications/node-connect-app/.npmrc

@@ -0,0 +1,2 @@
+@sentry:registry=http://127.0.0.1:4873
+@sentry-internal:registry=http://127.0.0.1:4873

dev-packages/e2e-tests/test-applications/node-connect-app/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "node-connect-app",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start": "ts-node src/app.ts",
+    "test": "playwright test",
+    "clean": "npx rimraf node_modules pnpm-lock.yaml",
+    "typecheck": "tsc",
+    "test:build": "pnpm install && pnpm run typecheck",
+    "test:assert": "pnpm test"
+  },
+  "dependencies": {
+    "@sentry/node": "latest || *",
+    "@sentry/types": "latest || *",
+    "@sentry/core": "latest || *",
+    "@sentry/utils": "latest || *",
+    "@sentry/opentelemetry": "latest || *",
+    "@types/node": "18.15.1",
+    "connect": "3.7.0",
+    "typescript": "4.9.5",
+    "ts-node": "10.9.1"
+  },
+  "devDependencies": {
+    "@sentry-internal/event-proxy-server": "link:../../../event-proxy-server",
+    "@playwright/test": "^1.38.1"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

dev-packages/e2e-tests/test-applications/node-connect-app/playwright.config.ts

@@ -0,0 +1,62 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+import { devices } from '@playwright/test';
+
+const connectPort = 3030;
+const eventProxyPort = 3031;
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './tests',
+  /* Maximum time one test can run for. */
+  timeout: 150_000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'list',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: `http://localhost:${connectPort}`,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: [
+    {
+      command: 'pnpm ts-node-script start-event-proxy.ts',
+      port: eventProxyPort,
+    },
+    {
+      command: 'pnpm start',
+      port: connectPort,
+    },
+  ],
+};
+
+export default config;