Hide skipped tests in CI

.cirrus.yml

@@ -25,4 +25,4 @@ task:
   tests_script:
     - export SKIP_IO_CAPTURE_TESTS=1
     - export CI_NO_IPV6=1
-    - sapi/cli/php run-tests.php -P -q -j2 -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP --offline --show-diff --show-slow 1000 --set-timeout 120 -d zend_extension=opcache.so
+    - sapi/cli/php run-tests.php -P -q -j2 -g FAIL,BORK,LEAK,XLEAK --offline --show-diff --show-slow 1000 --set-timeout 120 -d zend_extension=opcache.so

.github/actions/test-linux/action.yml

@@ -1,5 +1,8 @@
 name: Test
 inputs:
+  testArtifacts:
+    default: null
+    required: false
   runTestsParameters:
     default: ''
     required: false
@@ -18,10 +21,17 @@ runs:
         export PDO_DBLIB_TEST_USER="pdo_test"
         export PDO_DBLIB_TEST_PASS="password"
         export SKIP_IO_CAPTURE_TESTS=1
+        export TEST_PHP_JUNIT=junit.out.xml
         sapi/cli/php run-tests.php -P -q ${{ inputs.runTestsParameters }} \
           -j$(/usr/bin/nproc) \
-          -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP \
+          -g FAIL,BORK,LEAK,XLEAK \
           --offline \
           --show-diff \
           --show-slow 1000 \
           --set-timeout 120
+    - uses: actions/upload-artifact@v3
+      if: always() && inputs.testArtifacts != null
+      with:
+        name: ${{ github.job }}_${{ inputs.testArtifacts }}
+        path: ${{ github.workspace }}/junit.out.xml
+        retention-days: 5

.github/actions/test-macos/action.yml

@@ -1,5 +1,8 @@
 name: Test
 inputs:
+  testArtifacts:
+    default: null
+    required: false
   runTestsParameters:
     default: ''
     required: false
@@ -11,10 +14,17 @@ runs:
         set -x
         export SKIP_IO_CAPTURE_TESTS=1
         export CI_NO_IPV6=1
+        export TEST_PHP_JUNIT=junit.out.xml
         sapi/cli/php run-tests.php -P -q ${{ inputs.runTestsParameters }} \
           -j$(sysctl -n hw.ncpu) \
-          -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP \
+          -g FAIL,BORK,LEAK,XLEAK \
           --offline \
           --show-diff \
           --show-slow 1000 \
           --set-timeout 120
+    - uses: actions/upload-artifact@v3
+      if: always() && inputs.testArtifacts != null
+      with:
+        name: ${{ github.job }}_${{ inputs.testArtifacts }}
+        path: ${{ github.workspace }}/junit.out.xml
+        retention-days: 5

.github/workflows/nightly.yml

@@ -67,11 +67,13 @@ jobs:
       - name: Test
         uses: ./.github/actions/test-linux
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}
           runTestsParameters: >-
             ${{ matrix.run_tests_parameters }}
       - name: Test Tracing JIT
         uses: ./.github/actions/test-linux
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Tracing JIT
           runTestsParameters: >-
             ${{ matrix.run_tests_parameters }}
             -d zend_extension=opcache.so
@@ -80,13 +82,15 @@ jobs:
       - name: Test OpCache
         uses: ./.github/actions/test-linux
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} OpCache
           runTestsParameters: >-
             ${{ matrix.run_tests_parameters }}
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1
       - name: Test Function JIT
         uses: ./.github/actions/test-linux
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Function JIT
           runTestsParameters: >-
             ${{ matrix.run_tests_parameters }}
             -d zend_extension=opcache.so
@@ -127,9 +131,12 @@ jobs:
         run: sudo make install
       - name: Test
         uses: ./.github/actions/test-macos
+        with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}
       - name: Test Tracing JIT
         uses: ./.github/actions/test-macos
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Tracing JIT
           runTestsParameters: >-
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1
@@ -138,13 +145,15 @@ jobs:
       - name: Test OpCache
         uses: ./.github/actions/test-macos
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} OpCache
           runTestsParameters: >-
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1
             -d opcache.protect_memory=1
       - name: Test Function JIT
         uses: ./.github/actions/test-macos
         with:
+          testArtifacts: ${{ matrix.branch.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Function JIT
           runTestsParameters: >-
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1

.github/workflows/push.yml

@@ -50,9 +50,12 @@ jobs:
         uses: ./.github/actions/setup-x64
       - name: Test
         uses: ./.github/actions/test-linux
+        with:
+          testArtifacts: ${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}
       - name: Test Tracing JIT
         uses: ./.github/actions/test-linux
         with:
+          testArtifacts: ${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Tracing JIT
           runTestsParameters: >-
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1
@@ -78,9 +81,12 @@ jobs:
         run: sudo make install
       - name: Test
         uses: ./.github/actions/test-macos
+        with:
+          testArtifacts: ${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}
       - name: Test Tracing JIT
         uses: ./.github/actions/test-macos
         with:
+          testArtifacts: ${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }} Tracing JIT
           runTestsParameters: >-
             -d zend_extension=opcache.so
             -d opcache.enable_cli=1

.gitignore

@@ -281,6 +281,7 @@ tmp-php.ini
 # GitHub actions cache
 # ------------------------------------------------------------------------------
 /branch-commit-cache.json
+/junit.out.xml
 
 # ------------------------------------------------------------------------------
 # Special cases to invert previous ignore patterns

azure/libmysqlclient_test.yml

@@ -34,7 +34,7 @@ steps:
       export REPORT_EXIT_STATUS=no
       rm -rf junit.xml | true
       sapi/cli/php run-tests.php -P -q \
-          -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP \
+          -g FAIL,BORK,LEAK,XLEAK \
           --offline --show-diff --show-slow 1000 --set-timeout 120 \
           ext/pdo_mysql
     displayName: 'Test ${{ parameters.configurationName }}'

azure/test.yml

@@ -18,7 +18,7 @@ steps:
       rm -rf junit.xml | true
       sapi/cli/php run-tests.php -P -q \
           -j$(/usr/bin/nproc) \
-          -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP \
+          -g FAIL,BORK,LEAK,XLEAK \
           --offline \
           --show-diff \
           --show-slow 1000 \

run-tests.php

@@ -2690,7 +2690,9 @@ function run_test(string $php, $file, array $env): string
             $orig_shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
             $diff = "# original source file: $orig_shortname\n" . $diff;
         }
-        show_file_block('diff', $diff);
+        if (!$SHOW_ONLY_GROUPS || array_intersect($restype, $SHOW_ONLY_GROUPS)) {
+            show_file_block('diff', $diff);
+        }
         if (strpos($log_format, 'D') !== false && file_put_contents($diff_filename, $diff) === false) {
             error("Cannot create test diff - $diff_filename");
         }

travis/test.sh

@@ -6,7 +6,7 @@ if [ -z "$ARM64" ]; then export JOBS=$(nproc); else export JOBS=16; fi
 
 export SKIP_IO_CAPTURE_TESTS=1
 ./sapi/cli/php run-tests.php -P \
-    -g "FAIL,XFAIL,BORK,WARN,LEAK,SKIP" --offline --show-diff --show-slow 1000 \
+    -g "FAIL,BORK,LEAK" --offline --show-diff --show-slow 1000 \
     --set-timeout 120 -j$JOBS \
     -d extension=`pwd`/modules/zend_test.so \
     -d zend_extension=`pwd`/modules/opcache.so \