Skip to content

Commit 0d31d89

Browse files
iluuu1994iluuu1994
committed
Add benchmarking to CI
1 parent f17cf2e commit 0d31d89

File tree

6 files changed

+333
-0
lines changed

6 files changed

+333
-0
lines changed

‎.github/workflows/push.yml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,3 +215,77 @@ jobs:
215215
run: .github/scripts/windows/build.bat
216216
- name: Test
217217
run: .github/scripts/windows/test.bat
218+
BENCHMARKING:
219+
name: BENCHMARKING
220+
runs-on: ubuntu-22.04
221+
steps:
222+
- name: git checkout
223+
uses: actions/checkout@v3
224+
with:
225+
fetch-depth: 0
226+
- name: apt
227+
run: |
228+
set -x
229+
sudo apt-get update
230+
sudo apt-get install \
231+
bison \
232+
re2c \
233+
locales \
234+
openssl \
235+
language-pack-de \
236+
libsqlite3-dev \
237+
libonig-dev \
238+
libc-client-dev \
239+
valgrind
240+
- name: ccache
241+
uses: hendrikmuhs/ccache-action@v1.2
242+
with:
243+
key: "${{github.job}}-${{hashFiles('main/php_version.h')}}"
244+
append-timestamp: false
245+
- name: ./configure
246+
run: |
247+
set -x
248+
./buildconf --force
249+
./configure \
250+
--disable-debug \
251+
--with-valgrind \
252+
--enable-opcache \
253+
--enable-option-checking=fatal \
254+
--prefix=/usr \
255+
--with-config-file-scan-dir=/etc/php.d \
256+
--with-mysqli=mysqlnd \
257+
--with-pdo-sqlite \
258+
--enable-mbstring \
259+
--enable-sockets \
260+
--with-openssl \
261+
--enable-werror
262+
- name: make
263+
run: make -j$(/usr/bin/nproc) >/dev/null
264+
- name: make install
265+
uses: ./.github/actions/install-linux
266+
- name: Prepare SSH key
267+
run: |
268+
mkdir -p ~/.ssh
269+
echo "${{ secrets.BENCHMARK_DATA_DEPLOY_KEY }}" > ~/.ssh/id_ed25519
270+
chmod 600 ~/.ssh/id_ed25519
271+
git config --global user.name "Benchmark"
272+
git config --global user.email "benchmark@php.net"
273+
- name: Setup
274+
run: |
275+
sudo service mysql start
276+
mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS wordpress"
277+
mysql -uroot -proot -e "CREATE USER 'wordpress'@'localhost' IDENTIFIED BY 'wordpress'; FLUSH PRIVILEGES;"
278+
mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'wordpress'@'localhost' WITH GRANT OPTION;"
279+
mkdir -p /etc/php.d
280+
echo zend_extension=opcache.so >> /etc/php.d/opcache.ini
281+
echo opcache.enable=1 >> /etc/php.d/opcache.ini
282+
echo opcache.enable_cli=1 >> /etc/php.d/opcache.ini
283+
- name: Benchmark
284+
run: php benchmark/benchmark.php true
285+
- name: Show diff
286+
if: github.event_name == 'pull_request'
287+
run: >-
288+
php benchmark/generate_diff.php
289+
${{ github.event.pull_request.head.sha }}
290+
$(git merge-base ${{ github.base_ref }} ${{ github.head_ref }})
291+
> $GITHUB_STEP_SUMMARY

‎benchmark/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/repos

‎benchmark/benchmark.php

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
<?php
2+
3+
require_once __DIR__ . '/shared.php';
4+
5+
$commitResult = ($argv[1] ?? 'false') === 'true';
6+
$phpCgi = $argv[2] ?? dirname(PHP_BINARY) . '/php-cgi';
7+
if (!file_exists($phpCgi)) {
8+
fwrite(STDERR, "php-cgi not found\n");
9+
exit(1);
10+
}
11+
12+
function main() {
13+
global $commitResult;
14+
15+
$data = [];
16+
$data['bench.php'] = runBench();
17+
$data['symfony_demo'] = runSymfonyDemo();
18+
$data['wordpress_6.2'] = runWordpress();
19+
$result = json_encode($data, JSON_PRETTY_PRINT) . "\n";
20+
21+
if ($commitResult) {
22+
commitResult($result);
23+
} else {
24+
fwrite(STDOUT, $result);
25+
}
26+
}
27+
28+
function commitResult(string $result) {
29+
$repo = __DIR__ . '/repos/data';
30+
cloneRepo($repo, 'git@github.com:iluuu1994/php-benchmark-data.git');
31+
runCommand(['git', 'pull', '--end-of-options', 'origin'], $repo);
32+
33+
$commitHash = getPhpSrcCommitHash();
34+
$dir = $repo . '/' . substr($commitHash, 0, 2) . '/' . $commitHash;
35+
$summaryFile = $dir . '/summary.json';
36+
if (!is_dir($dir)) {
37+
mkdir($dir, 0755, true);
38+
}
39+
file_put_contents($summaryFile, $result);
40+
41+
runCommand(['git', 'add', '--end-of-options', $summaryFile], $repo);
42+
runCommand(['git', 'commit', '--allow-empty', '-m', 'Add result for php/php-src@' . $commitHash, '--author', 'Benchmark <benchmark@php.net>'], $repo);
43+
runCommand(['git', 'push'], $repo);
44+
}
45+
46+
function getPhpSrcCommitHash(): string {
47+
$result = runCommand(['git', 'log', '--pretty=format:%H', '-n', '1'], dirname(__DIR__));
48+
return $result->stdout;
49+
}
50+
51+
function runBench(): array {
52+
$process = runValgrindPhpCgiCommand([dirname(__DIR__) . '/Zend/bench.php']);
53+
return ['instructions' => extractInstructionsFromValgrindOutput($process->stderr)];
54+
}
55+
56+
function runSymfonyDemo(): array {
57+
$dir = __DIR__ . '/repos/symfony-demo-2.2.3';
58+
cloneRepo($dir, 'git@github.com:iluuu1994/symfony-demo-2.2.3.git');
59+
runPhpCommand([$dir . '/bin/console', 'cache:clear']);
60+
runPhpCommand([$dir . '/bin/console', 'cache:warmup']);
61+
$process = runValgrindPhpCgiCommand(['-T1,1', $dir . '/public/index.php']);
62+
return ['instructions' => extractInstructionsFromValgrindOutput($process->stderr)];
63+
}
64+
65+
function runWordpress(): array {
66+
$dir = __DIR__ . '/repos/wordpress-6.2';
67+
cloneRepo($dir, 'git@github.com:iluuu1994/wordpress-6.2.git');
68+
69+
/* FIXME: It might be better to use a stable version of PHP for this command because we can't
70+
* easily alter the phar file */
71+
runPhpCommand([
72+
'-d error_reporting=0',
73+
'wp-cli.phar',
74+
'core',
75+
'install',
76+
'--url=wordpress.local',
77+
'--title="Wordpress"',
78+
'--admin_user=wordpress',
79+
'--admin_password=wordpress',
80+
'--admin_email=benchmark@php.net',
81+
], $dir);
82+
83+
// Warmup
84+
runPhpCommand([$dir . '/index.php'], $dir);
85+
$process = runValgrindPhpCgiCommand(['-T1,1', $dir . '/index.php'], $dir);
86+
return ['instructions' => extractInstructionsFromValgrindOutput($process->stderr)];
87+
}
88+
89+
function runPhpCommand(array $args, ?string $cwd = null): ProcessResult {
90+
return runCommand([PHP_BINARY, ...$args], $cwd);
91+
}
92+
93+
function runValgrindPhpCgiCommand(array $args, ?string $cwd = null): ProcessResult {
94+
global $phpCgi;
95+
return runCommand([
96+
'valgrind',
97+
'--tool=callgrind',
98+
'--dump-instr=yes',
99+
'--callgrind-out-file=/dev/null',
100+
'--',
101+
$phpCgi,
102+
'-d max_execution_time=0',
103+
...$args,
104+
]);
105+
}
106+
107+
function extractInstructionsFromValgrindOutput(string $output): ?string {
108+
preg_match("(==[0-9]+== Events : Ir\n==[0-9]+== Collected : (?<instructions>[0-9]+))", $output, $matches);
109+
return $matches['instructions'] ?? null;
110+
}
111+
112+
main();

‎benchmark/docker-compose.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version: "3.8"
2+
services:
3+
wordpress_db:
4+
image: mysql:8.0
5+
ports:
6+
- "3306:3306"
7+
environment:
8+
MYSQL_ROOT_PASSWORD: root
9+
MYSQL_DATABASE: wordpress
10+
MYSQL_USER: wordpress
11+
MYSQL_PASSWORD: wordpress

‎benchmark/generate_diff.php

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<?php
2+
3+
require_once __DIR__ . '/shared.php';
4+
5+
function main(?string $headCommitHash, ?string $baseCommitHash) {
6+
if ($headCommitHash === null || $baseCommitHash === null) {
7+
fwrite(STDERR, "Usage: php generate_diff.php HEAD_COMMIT_HASH BASE_COMMIT_HASH\n");
8+
exit(1);
9+
}
10+
11+
$repo = __DIR__ . '/repos/data';
12+
cloneRepo($repo, 'git@github.com:iluuu1994/php-benchmark-data.git');
13+
$headSummaryFile = $repo . '/' . substr($headCommitHash, 0, 2) . '/' . $headCommitHash . '/summary.json';
14+
$baseSummaryFile = $repo . '/' . substr($baseCommitHash, 0, 2) . '/' . $baseCommitHash . '/summary.json';
15+
if (!file_exists($headSummaryFile)) {
16+
return "Head commit '$headCommitHash' not found\n";
17+
}
18+
if (!file_exists($baseSummaryFile)) {
19+
return "Base commit '$baseCommitHash' not found\n";
20+
}
21+
$headSummary = json_decode(file_get_contents($headSummaryFile), true);
22+
$baseSummary = json_decode(file_get_contents($baseSummaryFile), true);
23+
24+
$headCommitHashShort = substr($headCommitHash, 0, 7);
25+
$baseCommitHashShort = substr($baseCommitHash, 0, 7);
26+
$output = "| Benchmark | Base ($baseCommitHashShort) | Head ($headCommitHashShort) | Diff |\n";
27+
$output .= "|---|---|---|---|\n";
28+
foreach ($headSummary as $name => $headBenchmark) {
29+
$baseInstructions = $baseSummary[$name]['instructions'] ?? null;
30+
$headInstructions = $headSummary[$name]['instructions'];
31+
$output .= "| $name | "
32+
. formatInstructions($baseInstructions) . " | "
33+
. formatInstructions($headInstructions) . " | "
34+
. formatDiff($baseInstructions, $headInstructions) . " |\n";
35+
}
36+
return $output;
37+
}
38+
39+
function formatInstructions(?int $instructions): string {
40+
if ($instructions === null) {
41+
return '-';
42+
}
43+
if ($instructions > 1e6) {
44+
return sprintf('%.0fM', $instructions / 1e6);
45+
} elseif ($instructions > 1e3) {
46+
return sprintf('%.0fK', $instructions / 1e3);
47+
} else {
48+
return (string) $instructions;
49+
}
50+
}
51+
52+
function formatDiff(?int $baseInstructions, int $headInstructions): string {
53+
if ($baseInstructions === null) {
54+
return '-';
55+
}
56+
$instructionDiff = $headInstructions - $baseInstructions;
57+
return sprintf('%.2f%%', $instructionDiff / $baseInstructions * 100);
58+
}
59+
60+
$headCommitHash = $argv[1] ?? null;
61+
$baseCommitHash = $argv[2] ?? null;
62+
$output = main($headCommitHash, $baseCommitHash);
63+
fwrite(STDOUT, $output);

‎benchmark/shared.php

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?php
2+
3+
class ProcessResult {
4+
public $stdout;
5+
public $stderr;
6+
}
7+
8+
function runCommand(array $args, ?string $cwd = null): ProcessResult {
9+
$cmd = implode(' ', array_map('escapeshellarg', $args));
10+
$pipes = null;
11+
$result = new ProcessResult();
12+
$descriptorSpec = [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']];
13+
fwrite(STDOUT, "> $cmd\n");
14+
$processHandle = proc_open($cmd, $descriptorSpec, $pipes, $cwd ?? getcwd(), null);
15+
16+
$stdin = $pipes[0];
17+
$stdout = $pipes[1];
18+
$stderr = $pipes[2];
19+
20+
fclose($stdin);
21+
22+
stream_set_blocking($stdout, false);
23+
stream_set_blocking($stderr, false);
24+
25+
$stdoutEof = false;
26+
$stderrEof = false;
27+
28+
do {
29+
$read = [$stdout, $stderr];
30+
$write = null;
31+
$except = null;
32+
33+
stream_select($read, $write, $except, 1, 0);
34+
35+
foreach ($read as $stream) {
36+
$chunk = fgets($stream);
37+
if ($stream === $stdout) {
38+
$result->stdout .= $chunk;
39+
} elseif ($stream === $stderr) {
40+
$result->stderr .= $chunk;
41+
}
42+
}
43+
44+
$stdoutEof = $stdoutEof || feof($stdout);
45+
$stderrEof = $stderrEof || feof($stderr);
46+
} while(!$stdoutEof || !$stderrEof);
47+
48+
fclose($stdout);
49+
fclose($stderr);
50+
51+
$statusCode = proc_close($processHandle);
52+
if ($statusCode !== 0) {
53+
fwrite(STDOUT, $result->stdout);
54+
fwrite(STDERR, $result->stderr);
55+
fwrite(STDERR, 'Exited with status code ' . $statusCode . "\n");
56+
exit($statusCode);
57+
}
58+
59+
return $result;
60+
}
61+
62+
function cloneRepo(string $path, string $url) {
63+
if (is_dir($path)) {
64+
return;
65+
}
66+
$dir = dirname($path);
67+
$repo = basename($path);
68+
if (!is_dir($dir)) {
69+
mkdir($dir, 0755, true);
70+
}
71+
runCommand(['git', 'clone', '-q', '--end-of-options', $url, $repo], dirname($path));
72+
}

0 commit comments

Comments
 (0)