From 94dea74621c8404d2a158ca5cecd41142fdd6a27 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Tue, 28 Feb 2023 15:10:39 +0100 Subject: [PATCH 1/4] ci: Add new metrics overhead app To hopefully be able to debug replay issues a bit better, and have a bit more representative checking. --- .../overhead-metrics/configs/ci/collect.ts | 8 +- .../overhead-metrics/configs/dev/collect.ts | 11 +- .../src/results/metrics-stats.ts | 4 + packages/overhead-metrics/src/scenarios.ts | 25 +++ .../test-apps/booking-app/index.html | 83 ++++++++ .../test-apps/booking-app/main.js | 177 ++++++++++++++++++ .../test-apps/booking-app/style.css | 135 +++++++++++++ .../test-apps/booking-app/with-replay.html | 100 ++++++++++ .../test-apps/booking-app/with-sentry.html | 91 +++++++++ 9 files changed, 626 insertions(+), 8 deletions(-) create mode 100644 packages/overhead-metrics/test-apps/booking-app/index.html create mode 100644 packages/overhead-metrics/test-apps/booking-app/main.js create mode 100644 packages/overhead-metrics/test-apps/booking-app/style.css create mode 100644 packages/overhead-metrics/test-apps/booking-app/with-replay.html create mode 100644 packages/overhead-metrics/test-apps/booking-app/with-sentry.html diff --git a/packages/overhead-metrics/configs/ci/collect.ts b/packages/overhead-metrics/configs/ci/collect.ts index 86709530de5f..88a510fabdf0 100644 --- a/packages/overhead-metrics/configs/ci/collect.ts +++ b/packages/overhead-metrics/configs/ci/collect.ts @@ -2,7 +2,7 @@ import type { Metrics } from '../../src/collector.js'; import { MetricsCollector } from '../../src/collector.js'; import type { NumberProvider } from '../../src/results/metrics-stats.js'; import { MetricsStats } from '../../src/results/metrics-stats.js'; -import { JankTestScenario } from '../../src/scenarios.js'; +import { BookingAppScenario } from '../../src/scenarios.js'; import { printStats } from '../../src/util/console.js'; import { latestResultFile } from './env.js'; @@ -26,9 +26,9 @@ const collector = new MetricsCollector({ headless: true, cpuThrottling: 2 }); const result = await collector.execute({ name: 'jank', scenarios: [ - new JankTestScenario('index.html'), - new JankTestScenario('with-sentry.html'), - new JankTestScenario('with-replay.html'), + new BookingAppScenario('index.html', 100), + new BookingAppScenario('with-sentry.html', 100), + new BookingAppScenario('with-replay.html', 100), ], runs: 10, tries: 10, diff --git a/packages/overhead-metrics/configs/dev/collect.ts b/packages/overhead-metrics/configs/dev/collect.ts index 5eafb89797cd..4b2ffbc5480a 100644 --- a/packages/overhead-metrics/configs/dev/collect.ts +++ b/packages/overhead-metrics/configs/dev/collect.ts @@ -1,7 +1,7 @@ import type { Metrics } from '../../src/collector.js'; import { MetricsCollector } from '../../src/collector.js'; import { MetricsStats } from '../../src/results/metrics-stats.js'; -import { JankTestScenario } from '../../src/scenarios.js'; +import { BookingAppScenario } from '../../src/scenarios.js'; import { printStats } from '../../src/util/console.js'; import { latestResultFile } from './env.js'; @@ -9,9 +9,12 @@ const collector = new MetricsCollector(); const result = await collector.execute({ name: 'dummy', scenarios: [ - new JankTestScenario('index.html'), - new JankTestScenario('with-sentry.html'), - new JankTestScenario('with-replay.html'), + new BookingAppScenario('index.html', 50), + new BookingAppScenario('with-sentry.html', 50), + new BookingAppScenario('with-replay.html', 50), + new BookingAppScenario('index.html', 500), + new BookingAppScenario('with-sentry.html', 500), + new BookingAppScenario('with-replay.html', 500), ], runs: 1, tries: 1, diff --git a/packages/overhead-metrics/src/results/metrics-stats.ts b/packages/overhead-metrics/src/results/metrics-stats.ts index 4c59cd2a5b14..2ccab6632905 100644 --- a/packages/overhead-metrics/src/results/metrics-stats.ts +++ b/packages/overhead-metrics/src/results/metrics-stats.ts @@ -51,6 +51,10 @@ export class MetricsStats { private static _filteredValues(numbers: number[]): number[] { numbers.sort((a, b) => a - b); + if (numbers.length < 1) { + return []; + } + const q1 = ss.quantileSorted(numbers, 0.25); const q3 = ss.quantileSorted(numbers, 0.75); const iqr = q3 - q1; diff --git a/packages/overhead-metrics/src/scenarios.ts b/packages/overhead-metrics/src/scenarios.ts index 027e8564c863..c57544ca1eeb 100644 --- a/packages/overhead-metrics/src/scenarios.ts +++ b/packages/overhead-metrics/src/scenarios.ts @@ -53,3 +53,28 @@ export class JankTestScenario implements Scenario { await new Promise(resolve => setTimeout(resolve, 12000)); } } + +export class BookingAppScenario implements Scenario { + public constructor(private _indexFile: string, private _count: number) {} + + /** + * + */ + public async run(_: playwright.Browser, page: playwright.Page): Promise { + let url = path.resolve(`./test-apps/booking-app/${this._indexFile}`); + assert(fs.existsSync(url)); + url = `file:///${url.replace(/\\/g, '/')}?count=${this._count}`; + console.log('Navigating to ', url); + await page.goto(url, { waitUntil: 'load', timeout: 60000 }); + + // Click "Update" + await page.click('#search button'); + + for (let i = 1; i < 10; i++) { + await page.click(`.result:nth-child(${i}) [data-select]`); + } + + // Wait for flushing, which we set to 2000ms + await new Promise(resolve => setTimeout(resolve, 2000)); + } +} diff --git a/packages/overhead-metrics/test-apps/booking-app/index.html b/packages/overhead-metrics/test-apps/booking-app/index.html new file mode 100644 index 000000000000..c72f8d9ebff0 --- /dev/null +++ b/packages/overhead-metrics/test-apps/booking-app/index.html @@ -0,0 +1,83 @@ + + + + Demo Booking Engine + + + + + + + + +
+
+

This is a test app.

+ +
+ +
+ +
+
+
+
+
+
+
+ + + + diff --git a/packages/overhead-metrics/test-apps/booking-app/main.js b/packages/overhead-metrics/test-apps/booking-app/main.js new file mode 100644 index 000000000000..c906d1c1fa45 --- /dev/null +++ b/packages/overhead-metrics/test-apps/booking-app/main.js @@ -0,0 +1,177 @@ +(function () { + const searchForm = document.querySelector('#search'); + + searchForm.addEventListener('submit', event => { + event.preventDefault(); + + updateOffers(); + }); + + const obs = new MutationObserver(function (mutations) { + console.log(mutations); + }); + + obs.observe(document.documentElement, { + attributes: true, + attributeOldValue: true, + characterData: true, + characterDataOldValue: true, + childList: true, + subtree: true, + }); +})(); + +function updateOffers() { + const list = document.querySelector('.result-list'); + + // Clear out existing children + for (let el of list.children) { + list.removeChild(el); + } + + // Add new children + // Allow to define children count via URL ?count=100 + const url = new URL(window.location.href); + const count = parseInt(url.searchParams.get('count') || 50); + for (let i = 0; i < count; i++) { + const el = document.createElement('div'); + el.classList.add('result'); + el.innerHTML = generateResult(); + + const id = crypto.randomUUID(); + el.setAttribute('id', id); + + addListeners(id, el); + + list.appendChild(el); + } +} + +function addListeners(id, el) { + el.querySelector('[data-long-text-open]').addEventListener('click', event => { + const parent = event.target.closest('.long-text'); + parent.setAttribute('data-show-long', ''); + }); + el.querySelector('[data-long-text-close]').addEventListener('click', event => { + const parent = event.target.closest('.long-text'); + parent.removeAttribute('data-show-long'); + }); + + // These are purposefully inefficient + el.querySelector('[data-select]').addEventListener('click', () => { + document.querySelectorAll('.result').forEach(result => { + if (result.getAttribute('id') === id) { + result.setAttribute('data-show-options', 'yes'); + } else { + result.setAttribute('data-show-options', 'no'); + } + }); + + // Do some more, extra expensive work + document.querySelectorAll('.select__price').forEach(el => { + el.setAttribute('js-is-checked', new Date().toISOString()); + el.setAttribute('js-is-checked-2', new Date().toISOString()); + el.setAttribute('js-is-checked-3', 'yes'); + el.setAttribute('js-is-checked-4', 'yes'); + el.setAttribute('js-is-checked-5', 'yes'); + el.setAttribute('js-is-checked-6', 'yes'); + }); + document.querySelectorAll('.tag').forEach(el => el.setAttribute('js-is-checked', 'yes')); + document.querySelectorAll('h3').forEach(el => el.setAttribute('js-is-checked', 'yes')); + }); +} + +const baseTitles = ['Cottage house', 'Cabin', 'Villa', 'House', 'Appartment', 'Cosy appartment']; +const baseBeds = ['2', '2+2', '4+2', '6+2', '6+4']; +const baseDescription = + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'; + +function generateResult() { + const title = `${getRandomItem(baseTitles)} ${Math.ceil(Math.random() * 20)}`; + const beds = getRandomItem(baseBeds); + const description = baseDescription + .split(' ') + .slice(Math.ceil(Math.random() * 10)) + .join(' '); + const price = 200 + Math.random() * 800; + + // Make short version of description + const descriptionShort = description.slice(0, 200); + const priceStr = new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD' }).format(price); + + const placeholders = { + title, + beds, + description, + descriptionShort, + priceStr, + }; + + return replacePlaceholders(template, placeholders); +} + +function getRandomItem(list) { + return list[Math.floor(Math.random() * list.length)]; +} + +function replacePlaceholders(str, placeholders) { + let replacedStr = str; + Object.keys(placeholders).forEach(placeholder => { + replacedStr = replacedStr.replaceAll(`{{${placeholder}}}`, placeholders[placeholder]); + }); + + return replacedStr; +} + +const template = `
+ {{title}} +
+ +
+
+

{{title}}

+ +
+ {{beds}} +
+
+ +
+
+ {{descriptionShort}} +
+ +
+ {{description}} + +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+
`; diff --git a/packages/overhead-metrics/test-apps/booking-app/style.css b/packages/overhead-metrics/test-apps/booking-app/style.css new file mode 100644 index 000000000000..917e2dce5c6b --- /dev/null +++ b/packages/overhead-metrics/test-apps/booking-app/style.css @@ -0,0 +1,135 @@ +html, +body { + margin: 0; + padding: 0; +} + +body { + color: #4a4a4a; + font-size: 1em; + font-weight: 400; + line-height: 1.5; + font-family: system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica Neue, Arial, sans-serif; + box-sizing: border-box; +} + +*, +*:before, +*:after { + font-family: inherit; + box-sizing: inherit; +} + +.header { + margin-bottom: 1.5rem; + padding: 3rem; +} + +.header h1 { + margin: 0 0 1em 0; +} + +.search-form { + margin-bottom: 1.5rem; +} + +.search-form-fields { + display: flex; + flex-basis: 0; + flex-grow: 5; + flex-shrink: 1; +} + +.result { + margin-bottom: 1em; + padding-bottom: 1em; + border-bottom: 1px solid #dbdbdb; + display: flex; +} + +.result-image { + width: 350px; + margin: 0 1.5em 0 0; + padding: 0; +} + +.result-image img { + display: block; +} + +.result-content h3 { + margin-top: 0; + margin-bottom: 0.5em; +} + +.tags { + display: flex; + gap: 0.5em; + flex-wrap: wrap; + margin-bottom: 1.5rem; +} + +.tag { + background-color: #f5f5f5; + border-radius: 4px; + color: #4a4a4a; + display: inline-flex; + font-size: 0.75rem; + line-height: 2; + justify-content: center; + padding-left: 0.75em; + padding-right: 0.75em; + white-space: nowrap; +} + +.long-text { + margin-bottom: 1.5rem; +} + +[data-long-text-open], +[data-long-text-close] { + all: unset; + font-weight: bold; + cursor: pointer; +} + +:not([data-show-long]) > .long-text__long { + display: none; +} + +[data-show-long] > .long-text__short { + display: none; +} + +.select { +} + +.select button { + font-size: 1.25em; + padding: 0.25em; + display: flex; + width: 100%; + gap: 0.25em; +} + +.select__price { + margin-left: auto; + display: flex; +} + +.price__amount { + font-weight: bold; +} + +.options { + display: flex; + gap: 0.5em; +} + +.result:not([data-show-options='yes']) .options { + display: none; +} + +.result[data-show-options='yes'] .select { + display: none; +} diff --git a/packages/overhead-metrics/test-apps/booking-app/with-replay.html b/packages/overhead-metrics/test-apps/booking-app/with-replay.html new file mode 100644 index 000000000000..4f5e8710c5be --- /dev/null +++ b/packages/overhead-metrics/test-apps/booking-app/with-replay.html @@ -0,0 +1,100 @@ + + + + Demo Booking Engine + + + + + + + + +
+
+

This is a test app.

+ +
+ +
+ +
+
+
+
+
+
+
+ + + + + + diff --git a/packages/overhead-metrics/test-apps/booking-app/with-sentry.html b/packages/overhead-metrics/test-apps/booking-app/with-sentry.html new file mode 100644 index 000000000000..c687efc247e0 --- /dev/null +++ b/packages/overhead-metrics/test-apps/booking-app/with-sentry.html @@ -0,0 +1,91 @@ + + + + Demo Booking Engine + + + + + + + + +
+
+

This is a test app.

+ +
+ +
+ +
+
+
+
+
+
+
+ + + + + + From c800954d47be38e522146c38d41dd62fdda4a7e3 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Tue, 28 Feb 2023 15:34:06 +0100 Subject: [PATCH 2/4] ci: Fix CLS capture --- packages/overhead-metrics/src/vitals/cls.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/overhead-metrics/src/vitals/cls.ts b/packages/overhead-metrics/src/vitals/cls.ts index dfd9152beb4f..3e1ab977fb86 100644 --- a/packages/overhead-metrics/src/vitals/cls.ts +++ b/packages/overhead-metrics/src/vitals/cls.ts @@ -12,12 +12,10 @@ class CLS { const observer = new PerformanceObserver((list) => { for (const entry of list.getEntries()) { - if (!entry.hadRecentInput) { - if (window.cumulativeLayoutShiftScore === undefined) { - window.cumulativeLayoutShiftScore = entry.value; - } else { - window.cumulativeLayoutShiftScore += entry.value; - } + if (window.cumulativeLayoutShiftScore === undefined) { + window.cumulativeLayoutShiftScore = entry.value; + } else if (!entry.hadRecentInput) { + window.cumulativeLayoutShiftScore += entry.value; } } }); From a781ff3d67d0507b1e9601e28d8655b8a6782e8b Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 1 Mar 2023 10:12:10 +0100 Subject: [PATCH 3/4] extend timeout --- packages/overhead-metrics/src/scenarios.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/overhead-metrics/src/scenarios.ts b/packages/overhead-metrics/src/scenarios.ts index c57544ca1eeb..0fa64467ae34 100644 --- a/packages/overhead-metrics/src/scenarios.ts +++ b/packages/overhead-metrics/src/scenarios.ts @@ -74,7 +74,7 @@ export class BookingAppScenario implements Scenario { await page.click(`.result:nth-child(${i}) [data-select]`); } - // Wait for flushing, which we set to 2000ms - await new Promise(resolve => setTimeout(resolve, 2000)); + // Wait for flushing, which we set to 2000ms - to be safe, we add 1s on top + await new Promise(resolve => setTimeout(resolve, 3000)); } } From 5dabd264d4dd0111a2297d96d9d4b45490376bf8 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 1 Mar 2023 10:21:19 +0100 Subject: [PATCH 4/4] add test command & inline styles --- packages/overhead-metrics/package.json | 1 + .../test-apps/booking-app/index.html | 138 +++++++++++++++++- .../test-apps/booking-app/style.css | 135 ----------------- .../test-apps/booking-app/with-replay.html | 138 +++++++++++++++++- .../test-apps/booking-app/with-sentry.html | 138 +++++++++++++++++- 5 files changed, 412 insertions(+), 138 deletions(-) delete mode 100644 packages/overhead-metrics/test-apps/booking-app/style.css diff --git a/packages/overhead-metrics/package.json b/packages/overhead-metrics/package.json index 5ec818d406c1..4d471a591740 100644 --- a/packages/overhead-metrics/package.json +++ b/packages/overhead-metrics/package.json @@ -10,6 +10,7 @@ "build": "tsc", "dev:collect": "ts-node-esm ./configs/dev/collect.ts", "dev:process": "ts-node-esm ./configs/dev/process.ts", + "dev:run:replay": "npx chrome ./test-apps/booking-app/with-replay.html", "ci:collect": "ts-node-esm ./configs/ci/collect.ts", "ci:process": "ts-node-esm ./configs/ci/process.ts", "fix": "run-s fix:eslint fix:prettier", diff --git a/packages/overhead-metrics/test-apps/booking-app/index.html b/packages/overhead-metrics/test-apps/booking-app/index.html index c72f8d9ebff0..e3972c61e4c0 100644 --- a/packages/overhead-metrics/test-apps/booking-app/index.html +++ b/packages/overhead-metrics/test-apps/booking-app/index.html @@ -6,7 +6,143 @@ - + diff --git a/packages/overhead-metrics/test-apps/booking-app/style.css b/packages/overhead-metrics/test-apps/booking-app/style.css deleted file mode 100644 index 917e2dce5c6b..000000000000 --- a/packages/overhead-metrics/test-apps/booking-app/style.css +++ /dev/null @@ -1,135 +0,0 @@ -html, -body { - margin: 0; - padding: 0; -} - -body { - color: #4a4a4a; - font-size: 1em; - font-weight: 400; - line-height: 1.5; - font-family: system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica Neue, Arial, sans-serif; - box-sizing: border-box; -} - -*, -*:before, -*:after { - font-family: inherit; - box-sizing: inherit; -} - -.header { - margin-bottom: 1.5rem; - padding: 3rem; -} - -.header h1 { - margin: 0 0 1em 0; -} - -.search-form { - margin-bottom: 1.5rem; -} - -.search-form-fields { - display: flex; - flex-basis: 0; - flex-grow: 5; - flex-shrink: 1; -} - -.result { - margin-bottom: 1em; - padding-bottom: 1em; - border-bottom: 1px solid #dbdbdb; - display: flex; -} - -.result-image { - width: 350px; - margin: 0 1.5em 0 0; - padding: 0; -} - -.result-image img { - display: block; -} - -.result-content h3 { - margin-top: 0; - margin-bottom: 0.5em; -} - -.tags { - display: flex; - gap: 0.5em; - flex-wrap: wrap; - margin-bottom: 1.5rem; -} - -.tag { - background-color: #f5f5f5; - border-radius: 4px; - color: #4a4a4a; - display: inline-flex; - font-size: 0.75rem; - line-height: 2; - justify-content: center; - padding-left: 0.75em; - padding-right: 0.75em; - white-space: nowrap; -} - -.long-text { - margin-bottom: 1.5rem; -} - -[data-long-text-open], -[data-long-text-close] { - all: unset; - font-weight: bold; - cursor: pointer; -} - -:not([data-show-long]) > .long-text__long { - display: none; -} - -[data-show-long] > .long-text__short { - display: none; -} - -.select { -} - -.select button { - font-size: 1.25em; - padding: 0.25em; - display: flex; - width: 100%; - gap: 0.25em; -} - -.select__price { - margin-left: auto; - display: flex; -} - -.price__amount { - font-weight: bold; -} - -.options { - display: flex; - gap: 0.5em; -} - -.result:not([data-show-options='yes']) .options { - display: none; -} - -.result[data-show-options='yes'] .select { - display: none; -} diff --git a/packages/overhead-metrics/test-apps/booking-app/with-replay.html b/packages/overhead-metrics/test-apps/booking-app/with-replay.html index 4f5e8710c5be..9c4e0da222a7 100644 --- a/packages/overhead-metrics/test-apps/booking-app/with-replay.html +++ b/packages/overhead-metrics/test-apps/booking-app/with-replay.html @@ -6,7 +6,143 @@ - + diff --git a/packages/overhead-metrics/test-apps/booking-app/with-sentry.html b/packages/overhead-metrics/test-apps/booking-app/with-sentry.html index c687efc247e0..dcef1b45ef3c 100644 --- a/packages/overhead-metrics/test-apps/booking-app/with-sentry.html +++ b/packages/overhead-metrics/test-apps/booking-app/with-sentry.html @@ -6,7 +6,143 @@ - +