perf: run flamegrill in CI (#2163)

* store flamegrill results to json

* run flamegrill in circleci

* remove cruft console.log

* store flamegrill stats to db

* correctly name flamegrill data in db

Author: miroslavstastny

.circleci/config.yml

@@ -60,6 +60,9 @@ jobs:
       - run:
           name: Circular Dependencies Tests
           command: yarn test:circulars
+      - run:
+          name: Flamegrill
+          command: yarn perf:test
       - run:
           name: Bundle Statistics (master only)
           command: |

.github/test-a-feature.md

@@ -265,11 +265,10 @@ Finding the right number of `iterations` is a balancing act between having a fas
 
 Run test and watch:
 ```
-cd packages/perf-test
-yarn just perf-test
+yarn perf:test
 ```
 
-After running `perf-test`, results can be viewed in the `packages/perf-test/dist` folder with the main entry file being `packages/perf-test/dist/perfCounts.html`.
+After running `perf:test`, results can be viewed in the `packages/perf-test/dist` folder with the main entry file being `packages/perf-test/dist/perfCounts.html`.
 
 There are more detailed commands as well (these must be run from `packages/perf-test` directory):
 

build/gulp/tasks/stats.ts

@@ -143,6 +143,34 @@ function readSummaryPerfStats() {
     .value()
 }
 
+function readFlamegrillStats() {
+  return require(paths.packageDist('perf-test', 'perfCounts.json'))
+}
+
+// 1. iterate over all perf-test results
+// 2. the ones which have filename are docsite perf examples
+//    -> use camelCase name (docsite perf examples convention)
+//    -> and merge yarn perf (summaryPerf) and yarn perf:test (flamegrill) data
+// 3. the others are perf-test only examples -> store
+function mergePerfStats(summaryPerfStats, perfTestStats) {
+  return _.transform(
+    perfTestStats,
+    (result, value, key: string) => {
+      const flamegrill = _.pick(value, ['profile.metrics', 'analysis', 'extended'])
+      if (value['extended'].filename) {
+        const docsiteFilename = _.camelCase(value['extended'].filename)
+        result[docsiteFilename] = {
+          ...summaryPerfStats[docsiteFilename],
+          flamegrill,
+        }
+      } else {
+        result[key.replace(/\./g, '_')] = { flamegrill } // mongodb does not allow dots in keys
+      }
+    },
+    {},
+  )
+}
+
 function readCurrentBundleStats() {
   return _.mapKeys(require(currentStatsFilePath), (value, key) => _.camelCase(key)) // mongodb does not allow dots in keys
 }
@@ -158,6 +186,9 @@ task('stats:save', async () => {
   )
   const bundleStats = readCurrentBundleStats()
   const perfStats = readSummaryPerfStats()
+  const flamegrillStats = readFlamegrillStats()
+
+  const mergedPerfStats = mergePerfStats(perfStats, flamegrillStats)
 
   const prUrl =
     process.env.CIRCLE_PULL_REQUEST ||
@@ -170,7 +201,7 @@ task('stats:save', async () => {
     build: process.env.BUILD_BUILDID || process.env.CIRCLE_BUILD_NUM,
     ...commandLineArgs, // allow command line overwrites
     bundleSize: bundleStats,
-    performance: perfStats,
+    performance: mergedPerfStats,
     ts: new Date(),
   }
 

package.json

@@ -14,6 +14,7 @@
     "now-build": "cross-env NOW=true yarn predeploy:docs",
     "perf": "cross-env PERF=true gulp perf --times=100",
     "perf:debug": "cross-env PERF=true gulp perf:debug --debug",
+    "perf:test": "yarn lerna run --stream --scope @fluentui/perf-test perf:test",
     "prettier": "prettier --list-different \"**/*.{ts,tsx}\"",
     "prettier:fix": "prettier --write \"**/*.{ts,tsx}\"",
     "precommit": "lint-staged",

packages/perf-test/package.json

@@ -11,7 +11,8 @@
   "scripts": {
     "build": "just-scripts build",
     "clean": "just-scripts clean",
-    "test": "just-scripts test"
+    "test": "just-scripts test",
+    "perf:test": "just-scripts perf-test"
   },
   "devDependencies": {
     "@fluentui/digest": "^0.41.1",

packages/perf-test/tasks/perf-test.ts

@@ -1,9 +1,22 @@
 import fs from 'fs'
 import path from 'path'
+import _ from 'lodash'
 import flamegrill, { CookResult, CookResults, ScenarioConfig, Scenarios } from 'flamegrill'
 
 import { generateUrl } from '@fluentui/digest'
 
+type ExtendedCookResult = CookResult & {
+  extended: {
+    kind: string
+    story: string
+    iterations: number
+    tpi?: number
+    fabricTpi?: number
+    filename?: number
+  }
+}
+type ExtendedCookResults = Record<string, ExtendedCookResult>
+
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 // TODO:
 //
@@ -84,13 +97,18 @@ export default async function getPerfRegressions() {
   const scenarioConfig: ScenarioConfig = { outDir, tempDir }
   const scenarioResults = await flamegrill.cook(scenarios, scenarioConfig)
 
-  const comment = createReport(stories, scenarioResults)
+  const extendedCookResults = extendCookResults(stories, scenarioResults)
+  fs.writeFileSync(
+    path.join(outDir, 'perfCounts.json'),
+    JSON.stringify(extendedCookResults, null, 2),
+  )
+
+  const comment = createReport(stories, extendedCookResults)
 
   // TODO: determine status according to perf numbers
   const status = 'success'
 
   console.log(`Perf evaluation status: ${status}`)
-  console.log(`Writing comment to file:\n${comment}`)
 
   // Write results to file
   fs.writeFileSync(path.join(outDir, 'perfCounts.html'), comment)
@@ -101,13 +119,35 @@ export default async function getPerfRegressions() {
   console.log(`##vso[task.setvariable variable=PerfCommentStatus;]${status}`)
 }
 
+function extendCookResults(stories, testResults: CookResults): ExtendedCookResults {
+  return _.mapValues(testResults, (testResult, resultKey) => {
+    const kind = getKindKey(resultKey)
+    const story = getStoryKey(resultKey)
+    const iterations = getIterations(stories, kind, story)
+    const tpi = getTpiResult(testResults, stories, kind, story) // || 'n/a'
+    const fabricTpi = getTpiResult(testResults, stories, kind, 'Fabric') // || ''
+
+    return {
+      ...testResult,
+      extended: {
+        kind,
+        story,
+        iterations,
+        tpi,
+        fabricTpi,
+        filename: stories[kind][story].filename,
+      },
+    }
+  })
+}
+
 /**
  * Create test summary based on test results.
  *
  * @param {CookResults} testResults
  * @returns {string}
  */
-function createReport(stories, testResults: CookResults): string {
+function createReport(stories, testResults: ExtendedCookResults): string {
   const report = ''
 
     // TODO: We can't do CI, measure baseline or do regression analysis until master & PR files are deployed and publicly accessible.
@@ -131,7 +171,7 @@ function createReport(stories, testResults: CookResults): string {
  * @param {boolean} showAll Show only significant results by default.
  * @returns {string}
  */
-function createScenarioTable(stories, testResults: CookResults, showAll: boolean): string {
+function createScenarioTable(stories, testResults: ExtendedCookResults, showAll: boolean): string {
   const resultsToDisplay = Object.keys(testResults)
     .filter(
       key =>
@@ -191,18 +231,27 @@ function createScenarioTable(stories, testResults: CookResults, showAll: boolean
     resultsToDisplay
       .map(resultKey => {
         const testResult = testResults[resultKey]
-        const kind = getKindKey(resultKey)
-        const story = getStoryKey(resultKey)
-        const iterations = getIterations(stories, kind, story)
-        const tpi = getTpiResult(testResults, stories, kind, story) || 'n/a'
-        const fabricTpi = getTpiResult(testResults, stories, kind, 'Fabric') || ''
+        const tpi = testResult.extended.tpi
+          ? linkifyResult(
+              testResult,
+              testResult.extended.tpi.toLocaleString('en', { maximumSignificantDigits: 2 }),
+              false,
+            )
+          : 'n/a'
+        const fabricTpi = testResult.extended.fabricTpi
+          ? linkifyResult(
+              testResult,
+              testResult.extended.fabricTpi.toLocaleString('en', { maximumSignificantDigits: 2 }),
+              false,
+            )
+          : ''
 
         return `<tr>
-            <td>${kind}</td>
-            <td>${story}</td>
+            <td>${testResult.extended.kind}</td>
+            <td>${testResult.extended.story}</td>
             <td>${fabricTpi}</td>
             <td>${tpi}</td>
-            <td>${iterations}</td>
+            <td>${testResult.extended.iterations}</td>
             <td>${getTicksResult(testResult, false)}</td>
            </tr>`
       })
@@ -223,23 +272,19 @@ function getStoryKey(resultKey: string): string {
   return story
 }
 
-function getTpiResult(testResults, stories, kind, story): string | undefined {
+function getTpiResult(testResults, stories, kind, story): number | undefined {
   let tpi = undefined
   if (stories[kind][story]) {
     const resultKey = `${kind}.${story}`
     const testResult = testResults[resultKey]
     const ticks = getTicks(testResult)
     const iterations = getIterations(stories, kind, story)
-    tpi =
-      ticks &&
-      iterations &&
-      (ticks / iterations).toLocaleString('en', { maximumSignificantDigits: 2 })
-    tpi = linkifyResult(testResult, tpi, false)
+    tpi = ticks && iterations && Math.round((ticks / iterations) * 100) / 100
   }
   return tpi
 }
 
-function getIterations(stories, kind, story) {
+function getIterations(stories, kind, story): number {
   // Give highest priority to most localized definition of iterations. Story => kind => default.
   return (
     stories[kind][story].iterations ||