From 9fc7f24b8ead1bf16528cf2017f82dde23636977 Mon Sep 17 00:00:00 2001 From: "Christoph M. Becker" Date: Fri, 13 Sep 2024 17:01:27 +0200 Subject: [PATCH 1/2] Document --FLAKY-- section of PHPTs The given example might not be the best one, since it actually wouldn't need the `--FLAKY--` section, but it's the only one we currently have. --- phpt_details.php | 23 +++++++++++++++++++++ sample_tests/flaky.php | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 sample_tests/flaky.php diff --git a/phpt_details.php b/phpt_details.php index 9066bc2f..44c8b835 100644 --- a/phpt_details.php +++ b/phpt_details.php @@ -33,6 +33,7 @@ --FILE-- | --FILEEOF-- | --FILE_EXTERNAL-- | --REDIRECTTEST--
[--CGI--]
[--XFAIL--]
+[--FLAKY--]
[--EXPECTHEADERS--]
--EXPECT-- | --EXPECTF-- | --EXPECTREGEX-- | --EXPECT_EXTERNAL-- | --EXPECTF_EXTERNAL-- | --EXPECTREGEX_EXTERNAL-- @@ -666,6 +667,28 @@

Example 1 (full): sample017.phpt

+
--FLAKY--
+
+

Description:
+This section identifies this test as one that occassionally fails. If the test +actually fails, it will be retried one more time, and that result will be reported. +The section should include a brief description of why the test is flaky. Reasons for +this include tests that rely on relatively precise timing, or +temporary disc states. Available as of PHP 8.1.22 and 8.2.9, respectively.

+

Please do NOT include a --FLAKY-- section without providing a text description for +the reason it is being used.

+

Required:
+No.

+

Test Script Support:
+run-tests.php

+

Format:
+A short plain text description of why this test is flaky.

+

Example 1 (snippet):
+

--FLAKY--
+This test frequently fails in CI

+

Example 1 (full): flaky.phpt

+
+
--EXPECTHEADERS--

Description:
diff --git a/sample_tests/flaky.php b/sample_tests/flaky.php new file mode 100644 index 00000000..e2dabea1 --- /dev/null +++ b/sample_tests/flaky.php @@ -0,0 +1,47 @@ + + +

+

Sample Test: sample001.phpt

+

Back to "PHPT Test File Layout"

+
--TEST--
+Test hrtime() aligns with microtime()
+--FLAKY--
+This test frequently fails in CI
+--FILE--
+<?php
+
+$m0 = microtime(true);
+$h0 = hrtime(true);
+for ($i = 0; $i < 1024*1024; $i++);
+$h1 = hrtime(true);
+$m1 = microtime(true);
+
+$d0 = ($m1 - $m0)*1000000000.0;
+$d1 = $h1 - $h0;
+
+/* Relative uncertainty. */
+$d = abs($d0 - $d1)/$d1;
+
+if ($d > 0.05) {
+    print "FAIL, $d";
+} else {
+    print "OK, $d";
+}
+
+?>
+--EXPECTF--
+OK, %f
+
+

Back to "PHPT Test File Layout"

+
+ + From 011aaa3a6718f7c045be41586315afb0e4606f29 Mon Sep 17 00:00:00 2001 From: "Christoph M. Becker" Date: Tue, 17 Sep 2024 15:00:33 +0200 Subject: [PATCH 2/2] Document "flaky" convention for SKIPIF sections --- phpt_details.php | 14 +++++++++- sample_tests/flakyif.php | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 sample_tests/flakyif.php diff --git a/phpt_details.php b/phpt_details.php index 44c8b835..b77fc903 100644 --- a/phpt_details.php +++ b/phpt_details.php @@ -117,7 +117,10 @@

Format:
PHP code enclosed by PHP tags. If the output of this scripts starts with "skip", the test is skipped. If the output starts with "xfail", the test is marked as -expected failure. The "xfail" convention is supported as of PHP 7.2.0.

+expected failure. If the output starts with "flaky", +the test is marked as flaky test. +The "xfail" convention is supported as of PHP 7.2.0. +The "flaky" convention is supported as of PHP 8.2.25 and PHP 8.3.13, respectively.

Example 1 (snippet):

--SKIPIF--
 <?php if (!extension_loaded("filter")) die("Skipped: filter extension required."); ?>
@@ -133,6 +136,15 @@ <?php if (getenv('SKIP_ASAN')) die('xfail Startup failure leak'); ?>

Example 3 (full): xfailif.phpt

+

Example 4 (snippet):
+

--SKIPIF--
+<?php
+if (getenv("GITHUB_ACTIONS") && PHP_OS_FAMILY === "Darwin") {
+    die("flaky Occasionally segfaults on macOS for unknown reasons");
+}
+>>
+

+

Example 4 (full): flakyif.phpt

--CONFLICTS--
diff --git a/sample_tests/flakyif.php b/sample_tests/flakyif.php new file mode 100644 index 00000000..a7ecaef1 --- /dev/null +++ b/sample_tests/flakyif.php @@ -0,0 +1,59 @@ + + +
+

Sample Test: sample001.phpt

+

Back to "PHPT Test File Layout"

+
--TEST--
+Phar::chmod
+--EXTENSIONS--
+phar
+--INI--
+phar.readonly=1
+phar.require_hash=0
+--SKIPIF--
+<?php
+if (getenv("GITHUB_ACTIONS") && PHP_OS_FAMILY === "Darwin") {
+    die("flaky Occasionally segfaults on macOS for unknown reasons");
+}
+?>
+--FILE--
+<?php
+$fname = __DIR__ . '/' . basename(__FILE__, '.php') . '.1.phar.php';
+$pname = 'phar://hio';
+$file = '<?php include "' . $pname . '/a.php"; __HALT_COMPILER(); ?>';
+
+$files = array();
+$files['a.php']   = '<?php echo "This is a\n"; include "'.$pname.'/b.php"; ?>';
+include 'files/phar_test.inc';
+try {
+    $a = new Phar($fname);
+    var_dump($a['a.php']->isExecutable());
+    $a['a.php']->chmod(0777);
+    var_dump($a['a.php']->isExecutable());
+    $a['a.php']->chmod(0666);
+    var_dump($a['a.php']->isExecutable());
+} catch (Exception $e) {
+    echo $e->getMessage() . "\n";
+}
+?>
+--CLEAN--
+<?php
+unlink(__DIR__ . '/' . basename(__FILE__, '.clean.php') . '.1.phar.php');
+?>
+--EXPECTF--
+bool(false)
+Cannot modify permissions for file "a.php" in phar "%s033a.1.phar.php", write operations are prohibited
+
+

Back to "PHPT Test File Layout"

+
+ +