5
5
const os = require ( 'os' ) ;
6
6
const fs = require ( 'fs' ) ;
7
7
const path = require ( 'path' ) ;
8
- const { Benchmark } = require ( 'benchmark' ) ;
9
8
10
9
const {
11
10
exec,
@@ -16,6 +15,7 @@ const {
16
15
readdirRecursive,
17
16
} = require ( './utils' ) ;
18
17
18
+ const NS_PER_SEC = 1e9 ;
19
19
const LOCAL = 'local' ;
20
20
21
21
function LOCAL_DIR ( ...paths ) {
@@ -93,12 +93,20 @@ function runBenchmark(benchmark, environments) {
93
93
const benches = environments . map ( environment => {
94
94
const module = require ( path . join ( environment . distPath , benchmark ) ) ;
95
95
benchmarkName = module . name ;
96
- return new Benchmark ( environment . revision , module . measure ) ;
96
+ return {
97
+ name : environment . revision ,
98
+ fn : module . measure ,
99
+ } ;
97
100
} ) ;
98
101
99
102
console . log ( '⏱️ ' + benchmarkName ) ;
100
103
for ( let i = 0 ; i < benches . length ; ++ i ) {
101
- benches [ i ] . run ( { async : false } ) ;
104
+ const bench = benches [ i ] ;
105
+ try {
106
+ bench . sample = run ( bench . fn ) ;
107
+ } catch ( e ) {
108
+ bench . error = e ;
109
+ }
102
110
process . stdout . write ( ' ' + cyan ( i + 1 ) + ' tests completed.\u000D' ) ;
103
111
}
104
112
console . log ( '\n' ) ;
@@ -107,14 +115,123 @@ function runBenchmark(benchmark, environments) {
107
115
console . log ( '' ) ;
108
116
}
109
117
118
+ // Clocks the time taken to execute a test per cycle (secs).
119
+ function clock ( count , fn ) {
120
+ const start = process . hrtime . bigint ( ) ;
121
+ for ( let i = count ; count !== 0 ; -- count ) {
122
+ fn ( ) ;
123
+ }
124
+ return Number ( process . hrtime . bigint ( ) - start ) ;
125
+ }
126
+
127
+ // Cycles a benchmark until a run `count` can be established.
128
+ function cycle ( initCount , fn ) {
129
+ // Resolve time span required to achieve a percent uncertainty of at most 1%.
130
+ // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html.
131
+ const minTime = 0.05 * NS_PER_SEC ;
132
+ let count = initCount ;
133
+ let clocked = 0 ;
134
+
135
+ while ( true ) {
136
+ clocked = clock ( count , fn ) ;
137
+
138
+ // Do we need to do another cycle?
139
+ if ( clocked >= minTime ) {
140
+ break ;
141
+ }
142
+
143
+ // Calculate how many more iterations it will take to achieve the `minTime`.
144
+ count += Math . ceil ( ( minTime - clocked ) / clocked * count ) ;
145
+ }
146
+
147
+ return count ;
148
+ }
149
+
150
+ // Computes stats on benchmark results.
151
+ function run ( fn ) {
152
+ const bench = this ;
153
+ const sample = [ ] ;
154
+
155
+ // The maximum time a benchmark is allowed to run before finishing.
156
+ const maxTime = 5 * NS_PER_SEC ;
157
+ // The minimum sample size required to perform statistical analysis.
158
+ const minSamples = 15 ;
159
+ // The default number of times to execute a test on a benchmark's first cycle.
160
+ const initCount = 10 ;
161
+
162
+ const count = cycle ( initCount , fn ) ;
163
+ clock ( count , fn ) ; // initial warm up
164
+ // If time permits, increase sample size to reduce the margin of error.
165
+ let elapsed = 0 ;
166
+ while ( sample . length < minSamples || elapsed < maxTime ) {
167
+ const clocked = clock ( count , fn ) ;
168
+ elapsed += clocked ;
169
+ // Compute the seconds per operation.
170
+ sample . push ( clocked / count ) ;
171
+ }
172
+
173
+ return sample ;
174
+ }
175
+
176
+ /*
177
+ * T-Distribution two-tailed critical values for 95% confidence.
178
+ * For more info see http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
179
+ */
180
+ var tTable = {
181
+ '1' : 12.706 , '2' : 4.303 , '3' : 3.182 , '4' : 2.776 , '5' : 2.571 , '6' : 2.447 ,
182
+ '7' : 2.365 , '8' : 2.306 , '9' : 2.262 , '10' : 2.228 , '11' : 2.201 , '12' : 2.179 ,
183
+ '13' : 2.16 , '14' : 2.145 , '15' : 2.131 , '16' : 2.12 , '17' : 2.11 , '18' : 2.101 ,
184
+ '19' : 2.093 , '20' : 2.086 , '21' : 2.08 , '22' : 2.074 , '23' : 2.069 , '24' : 2.064 ,
185
+ '25' : 2.06 , '26' : 2.056 , '27' : 2.052 , '28' : 2.048 , '29' : 2.045 , '30' : 2.042 ,
186
+ 'infinity' : 1.96
187
+ } ;
188
+
110
189
function beautifyBenchmark ( results ) {
111
- const benches = results . map ( result => ( {
112
- name : result . name ,
113
- error : result . error ,
114
- ops : result . hz ,
115
- deviation : result . stats . rme ,
116
- numRuns : result . stats . sample . length ,
117
- } ) ) ;
190
+ const benches = results . map ( result => {
191
+ const sample = result . sample || [ ] ;
192
+
193
+ // Compute the sample mean (estimate of the population mean).
194
+ let mean = 0 ;
195
+ for ( const x of sample ) {
196
+ mean += x ;
197
+ }
198
+ mean /= sample . length ;
199
+ mean = mean || 0 ;
200
+
201
+ // Compute the sample variance (estimate of the population variance).
202
+ let variance = 0 ;
203
+ for ( const x of sample ) {
204
+ variance += Math . pow ( x - mean , 2 ) ;
205
+ }
206
+ variance /= ( sample . length - 1 ) ;
207
+ variance = variance || 0 ;
208
+
209
+ // Compute the sample standard deviation (estimate of the population standard deviation).
210
+ const sd = Math . sqrt ( variance ) ;
211
+
212
+ // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean).
213
+ const sem = sd / Math . sqrt ( sample . length ) ;
214
+
215
+ // Compute the degrees of freedom.
216
+ const df = sample . length - 1 ;
217
+
218
+ // Compute the critical value.
219
+ const critical = tTable [ Math . round ( df ) || 1 ] || tTable . infinity ;
220
+
221
+ // Compute the margin of error.
222
+ const moe = sem * critical ;
223
+
224
+ // The relative margin of error (expressed as a percentage of the mean).
225
+ const rme = ( moe / mean ) * 100 || 0 ;
226
+
227
+ return {
228
+ name : result . name ,
229
+ error : result . error ,
230
+ ops : NS_PER_SEC / mean ,
231
+ deviation : rme ,
232
+ numRuns : sample . length ,
233
+ } ;
234
+ } ) ;
118
235
119
236
const nameMaxLen = maxBy ( benches , ( { name } ) => name . length ) ;
120
237
const opsTop = maxBy ( benches , ( { ops } ) => ops ) ;
0 commit comments