feat(benchmark): calculate relative margin of error and simplify report

Author: jeffbcross

benchmark/web/bp.js

@@ -13,37 +13,27 @@ bp.runState = {
   timesPerAction: {}
 };
 
-bp.Statistics.getStabilityOfSample = function (sample, confidenceRange) {
-  var inRange = 0;
-  sample.forEach(function(x) {
-    inRange += x <= confidenceRange[1] && x >= confidenceRange[0] ? 1 : 0;
-  })
-  return Math.round((inRange / sample.length) * 100) / 100;
-
-};
-
-bp.Statistics.getConfidenceRange = function(mean, confidenceInterval) {
-  return [
-    Math.round((mean - confidenceInterval) * 100) / 100,
-    Math.round((mean + confidenceInterval) * 100) / 100
-  ];
-};
+bp.Statistics.getMean = function (sample) {
+  var total = 0;
+  sample.forEach(function(x) { total += x; });
+  return total / sample.length;
+}
 
 bp.Statistics.calculateConfidenceInterval = function(standardDeviation, sampleSize) {
   var standardError = standardDeviation / Math.sqrt(sampleSize);
-  var marginOfError = bp.Statistics.criticalValue * standardError;
-  marginOfError = Math.round(marginOfError * 100) / 100;
-  return marginOfError;
+  return bp.Statistics.criticalValue * standardError;
 };
 
-bp.Statistics.calculateStandardDeviation = function(sample) {
-  var mean = 0;
-  var deviation = 0;
-  sample.forEach(function(x) {
-    mean += x;
-  });
-  mean = mean / sample.length;
+bp.Statistics.calculateRelativeMarginOfError = function (marginOfError, mean) {
+  /*
+   * Converts absolute margin of error to a relative margin of error by
+   * converting it to a percentage of the mean.
+   */
+  return (marginOfError / mean);
+};
 
+bp.Statistics.calculateStandardDeviation = function(sample, mean) {
+  var deviation = 0;
   sample.forEach(function(x) {
     deviation += Math.pow(x - mean, 2);
   });
@@ -194,27 +184,20 @@ bp.generateReportModel = function (rawModel) {
   rawModel.gcTimes = rawModel.gcTimes.join('<br>'),
   rawModel.garbageTimes = rawModel.garbageTimes.join('<br>'),
   rawModel.retainedTimes = rawModel.retainedTimes.join('<br>')
+  rawModel.timesConfidenceInterval = (rawModel.timesConfidenceInterval || 0).toFixed(2);
   return rawModel;
 };
 
 bp.generateReportPartial = function(model) {
   return bp.infoTemplate(model);
 };
 
-bp.getAverage = function (times, gcTimes, garbageTimes, retainedTimes) {
-  var timesAvg = 0;
-  var gcAvg = 0;
-  var garbageAvg = 0;
-  var retainedAvg = 0;
-  times.forEach(function(x) { timesAvg += x; });
-  gcTimes.forEach(function(x) { gcAvg += x; });
-  garbageTimes.forEach(function(x) { garbageAvg += x; });
-  retainedTimes.forEach(function(x) { retainedAvg += x; });
+bp.getAverages = function (times, gcTimes, garbageTimes, retainedTimes) {
   return {
-    gcTime: gcAvg / gcTimes.length,
-    time: timesAvg / times.length,
-    garbage: garbageAvg / garbageTimes.length,
-    retained: retainedAvg / retainedTimes.length
+    gcTime: bp.Statistics.getMean(gcTimes),
+    time: bp.Statistics.getMean(times),
+    garbage: bp.Statistics.getMean(garbageTimes),
+    retained: bp.Statistics.getMean(retainedTimes)
   };
 };
 
@@ -269,8 +252,7 @@ bp.calcStats = function() {
         tpa = bp.getTimesPerAction(stepName),
         reportModel,
         avg,
-        timesConfidenceInterval,
-        timesConfidenceRange;
+        timesConfidenceInterval;
 
     bp.updateTimes(tpa, tpa.nextEntry, 'gcTimes', gcTimeForStep);
     bp.updateTimes(tpa, tpa.nextEntry, 'garbageTimes', garbageTimeForStep / 1e3);
@@ -279,27 +261,22 @@ bp.calcStats = function() {
 
     tpa.nextEntry++;
     tpa.nextEntry %= bp.runState.numSamples;
-    avg = bp.getAverage(
+    avg = bp.getAverages(
         tpa.times,
         tpa.gcTimes,
         tpa.garbageTimes,
         tpa.retainedTimes);
 
     timesConfidenceInterval = bp.Statistics.calculateConfidenceInterval(
-        bp.Statistics.calculateStandardDeviation(tpa.times),
+        bp.Statistics.calculateStandardDeviation(tpa.times, avg.time),
         tpa.times.length
     );
-    timesConfidenceRange = bp.Statistics.getConfidenceRange(
-        avg.time,
-        timesConfidenceInterval
-    );
+
     reportModel = bp.generateReportModel({
       name: stepName,
       avg: avg,
       times: tpa.fmtTimes,
-      timesConfidenceInterval: timesConfidenceInterval,
-      timesConfidenceRange: timesConfidenceRange,
-      timesStability: bp.Statistics.getStabilityOfSample(tpa.times, timesConfidenceRange) * 100,
+      timesRelativeMarginOfError: bp.Statistics.calculateRelativeMarginOfError(timesConfidenceInterval, avg.time),
       gcTimes: tpa.fmtGcTimes,
       garbageTimes: tpa.fmtGarbageTimes,
       retainedTimes: tpa.fmtRetainedTimes

benchmark/web/bp.spec.js

@@ -37,35 +37,28 @@ describe('bp', function() {
   describe('.Statistics', function() {
     describe('.calculateConfidenceInterval()', function() {
       it('should provide the correct confidence interval', function() {
-        expect(bp.Statistics.calculateConfidenceInterval(30, 1000)).toBe(1.86);
+        expect(bp.Statistics.calculateConfidenceInterval(30, 1000)).toBe(1.859419264179007);
       });
     });
 
 
-    describe('.calculateStandardDeviation()', function() {
-      it('should provide the correct standardDeviation for the provided sample', function() {
-        expect(bp.Statistics.calculateStandardDeviation([
-          2,4,4,4,5,5,7,9
-        ])).toBe(2);
-      });
+    describe('.calculateRelativeMarginOfError()', function() {
+      expect(bp.Statistics.calculateRelativeMarginOfError(1.85, 5)).toBe(0.37);
     });
 
 
-    describe('.getConfidenceRange()', function() {
-      it('should return an array of low and high confidence range', function() {
-        expect(bp.Statistics.getConfidenceRange(100, 1.5)).toEqual([98.5,101.5]);
+    describe('.getMean()', function() {
+      it('should return the mean for a given sample', function() {
+        expect(bp.Statistics.getMean([1,2,5,4])).toBe(3);
       });
-
-
-      it('should round values to 2 decimal points', function() {
-        expect(bp.Statistics.getConfidenceRange(100,1.111111)).toEqual([100-1.11,100+1.11]);
-      })
     });
 
 
-    describe('.getStabilityOfSample()', function() {
-      it('should return the percentage of samples that fall within the provided range', function() {
-        expect(bp.Statistics.getStabilityOfSample([0,5,5,10],[5,6])).toBe(0.5);
+    describe('.calculateStandardDeviation()', function() {
+      it('should provide the correct standardDeviation for the provided sample and mean', function() {
+        expect(bp.Statistics.calculateStandardDeviation([
+          2,4,4,4,5,5,7,9
+        ], 5)).toBe(2);
       });
     });
   });
@@ -396,9 +389,9 @@ describe('bp', function() {
   });
 
 
-  describe('.getAverage()', function() {
+  describe('.getAverages()', function() {
     it('should return the average of a set of numbers', function() {
-      expect(bp.getAverage([100,0,50,75,25], [2,4,2,4,3], [1,2],[3,4])).toEqual({
+      expect(bp.getAverages([100,0,50,75,25], [2,4,2,4,3], [1,2],[3,4])).toEqual({
         gcTime: 3,
         time: 50,
         garbage: 1.5,
@@ -450,8 +443,8 @@ describe('bp', function() {
     });
 
 
-    it('should call getAverage() with the correct info', function() {
-      var spy = spyOn(bp, 'getAverage').andCallThrough();
+    it('should call getAverages() with the correct info', function() {
+      var spy = spyOn(bp, 'getAverages').andCallThrough();
       bp.calcStats();
       expect(spy).toHaveBeenCalledWith([ 3, 7, 5 ], [ 1, 3, 2 ], [50,50,0.2], [25,25,0.1]);
     });
@@ -485,7 +478,8 @@ describe('bp', function() {
         times: ['1','2'],
         gcTimes: ['4','5'],
         garbageTimes: ['6','7'],
-        retainedTimes: ['7','8']
+        retainedTimes: ['7','8'],
+        timesConfidenceInterval: 0.5555
       })).toEqual({
         name : 'Some Step',
         avg : {
@@ -498,7 +492,8 @@ describe('bp', function() {
         times : '1<br>2',
         gcTimes : '4<br>5',
         garbageTimes : '6<br>7',
-        retainedTimes : '7<br>8'
+        retainedTimes : '7<br>8',
+        timesConfidenceInterval: '0.56'
       });
     });
   });

benchmark/web/tree.html

@@ -68,15 +68,7 @@
             <tr class="sampleContainer">
               <td><%= name %></td>
               <td class="average">
-                test:<%= avg.time %>ms
-                <br>
-                times within range: <%= timesStability %>%
-                <br>
-                confidence interval: <%= timesConfidenceInterval %>
-                <br>
-                confidence range:
-                  <%= timesConfidenceRange[0] %> -
-                  <%= timesConfidenceRange[1] %>
+                test:<%= avg.time %>ms &plusmn; <%- Math.round(timesRelativeMarginOfError * 100) %>%
                 <br>
                 gc:<%= avg.gcTime %>ms
                 <br>