Add retrying mechanism for IO tests in run-tests.php

Author: iluuu1994

run-tests.php

@@ -1851,11 +1851,16 @@ function run_test(string $php, $file, array $env): string
         $skipCache = new SkipCache($enableSkipCache, $cfg['keep']['skip']);
     }
 
+    $php = escapeshellarg($php);
+
+    $retriable = true;
+    $retried = false;
+retry:
+
     $temp_filenames = null;
     $org_file = $file;
     $orig_php = $php;
 
-    $php = escapeshellarg($php);
     $php_cgi = $env['TEST_PHP_CGI_EXECUTABLE'] ?? null;
     $phpdbg = $env['TEST_PHPDBG_EXECUTABLE'] ?? null;
 
@@ -1891,8 +1896,11 @@ function run_test(string $php, $file, array $env): string
 
     $tested = $test->getName();
 
-    if ($num_repeats > 1 && $test->hasSection('FILE_EXTERNAL')) {
-        return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable');
+    if ($test->hasSection('FILE_EXTERNAL')) {
+        $retriable = false;
+        if ($num_repeats > 1) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable');
+        }
     }
 
     if ($test->hasSection('CAPTURE_STDIO')) {
@@ -1918,6 +1926,7 @@ function run_test(string $php, $file, array $env): string
         }
         $php = escapeshellarg($php_cgi) . ' -C ';
         $uses_cgi = true;
+        $retriable = false;
         if ($num_repeats > 1) {
             return skip_test($tested, $tested_file, $shortname, 'CGI does not support --repeat');
         }
@@ -1935,20 +1944,18 @@ function run_test(string $php, $file, array $env): string
         } else {
             return skip_test($tested, $tested_file, $shortname, 'phpdbg not available');
         }
+        $retriable = false;
         if ($num_repeats > 1) {
             return skip_test($tested, $tested_file, $shortname, 'phpdbg does not support --repeat');
         }
     }
 
-    if ($num_repeats > 1) {
-        if ($test->hasSection('CLEAN')) {
-            return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable');
-        }
-        if ($test->hasSection('STDIN')) {
-            return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable');
-        }
-        if ($test->hasSection('CAPTURE_STDIO')) {
-            return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable');
+    foreach (['CLEAN', 'STDIN', 'CAPTURE_STDIO'] as $section) {
+        if ($test->hasSection($section)) {
+            $retriable = false;
+            if ($num_repeats > 1) {
+                return skip_test($tested, $tested_file, $shortname, "Test with $section might not be repeatable");
+            }
         }
     }
 
@@ -2113,7 +2120,7 @@ function run_test(string $php, $file, array $env): string
             // even though all the files are re-created.
             $ini_settings['opcache.validate_timestamps'] = '0';
         }
-    } else if ($num_repeats > 1) {
+    } else if ($num_repeats > 1 || $retried) {
         // Make sure warnings still show up on the second run.
         $ini_settings['opcache.record_warnings'] = '1';
     }
@@ -2140,8 +2147,11 @@ function run_test(string $php, $file, array $env): string
         }
         settings2array(preg_split("/[\n\r]+/", $ini), $ini_settings);
 
-        if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) {
-            return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable');
+        if (isset($ini_settings['opcache.opt_debug_level'])) {
+            $retriable = false;
+            if ($num_repeats > 1) {
+                return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable');
+            }
         }
     }
 
@@ -2672,6 +2682,10 @@ function run_test(string $php, $file, array $env): string
 
         $wanted_re = null;
     }
+    if (!$passed && !$retried && $retriable && error_may_be_retried($output)) {
+        $retried = true;
+        goto retry;
+    }
 
     if ($passed) {
         if (!$cfg['keep']['php'] && !$leaked) {
@@ -2702,6 +2716,9 @@ function run_test(string $php, $file, array $env): string
             } elseif ($test->hasSection('XLEAK')) {
                 $warn = true;
                 $info = " (warn: XLEAK section but test passes)";
+            } elseif ($retried) {
+                $warn = true;
+                $info = " (warn: Test passed on retry attempt)";
             } else {
                 show_result("PASS", $tested, $tested_file, '', $temp_filenames);
                 $junit->markTestAs('PASS', $shortname, $tested);
@@ -2845,6 +2862,11 @@ function run_test(string $php, $file, array $env): string
     return $restype[0] . 'ED';
 }
 
+function error_may_be_retried(string $output): bool
+{
+    return preg_match('((timed out)|(connection refused))i', $output) === 1;
+}
+
 /**
  * Map "Zend OPcache" to "opcache" and convert all ext names to lowercase.
  */