From d96f31a52603a7c380852b3d83b01c88b8df4320 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 03:41:58 +0000 Subject: [PATCH 1/3] Split the Behat tests with code coverage across runners Co-Authored-By: Claude Opus 5.5 --- .github/workflows/reusable-functional.yml | 59 +++++++++++---- .github/workflows/reusable-testing.yml | 88 ++++++++++++++++++++++- 2 files changed, 130 insertions(+), 17 deletions(-) diff --git a/.github/workflows/reusable-functional.yml b/.github/workflows/reusable-functional.yml index ee3dfa3..84210ac 100644 --- a/.github/workflows/reusable-functional.yml +++ b/.github/workflows/reusable-functional.yml @@ -24,6 +24,21 @@ on: type: boolean required: false default: false + coverage-driver: + description: 'Code coverage driver, either `pcov` or `xdebug`.' + type: string + required: false + default: 'pcov' + shard: + description: 'Share of the scenarios to run, as `/`, for example `2/3`. Empty to run all of them.' + type: string + required: false + default: '' + parallel: + description: 'Number of parallel workers on the runner.' + type: number + required: false + default: 4 os: description: 'Runner to use. Defaults to ubuntu-22.04, or the RUNNERS_NAME repository variable when set.' type: string @@ -40,7 +55,7 @@ jobs: # anything omitted here is not shown anywhere. The database version is spelled # out because "MySQL" alone rendered the mysql-8.0 and mysql-8.4 legs # identically. - name: Behat | PHP ${{ inputs.php }} | WP ${{ inputs.wp }} | ${{ inputs.dbtype == 'sqlite' && 'SQLite' || inputs.mysql || 'MySQL' }}${{ inputs.object_cache == 'sqlite' && ' (Obj Cache)' || '' }}${{ inputs.coverage && ' (with coverage)' || '' }}${{ startsWith( inputs.os, 'windows' ) && ' (Windows)' || '' }}${{ startsWith( inputs.os, 'macos' ) && ' (macOS)' || '' }} + name: Behat | PHP ${{ inputs.php }} | WP ${{ inputs.wp }} | ${{ inputs.dbtype == 'sqlite' && 'SQLite' || inputs.mysql || 'MySQL' }}${{ inputs.object_cache == 'sqlite' && ' (Obj Cache)' || '' }}${{ inputs.coverage && format( ' (with coverage{0})', inputs.shard && format( ', {0}', inputs.shard ) || '' ) || '' }}${{ startsWith( inputs.os, 'windows' ) && ' (Windows)' || '' }}${{ startsWith( inputs.os, 'macos' ) && ' (macOS)' || '' }} # Repositories with a heavy Behat suite can point the default Linux legs at a # larger runner by setting the `RUNNERS_NAME` repository variable, without # having to fork this workflow. Explicit macOS/Windows legs are unaffected. @@ -166,9 +181,13 @@ jobs: uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 with: php-version: '${{ inputs.php }}' - ini-values: zend.assertions=1, error_reporting=-1, display_errors=On + # PCOV only instruments the files below pcov.directory. Left unset, it would pick the + # working directory of every `wp` process the scenarios run, which is a temporary + # WordPress install, and collect nothing. Xdebug instead instruments every file it + # loads, all of WordPress included, which makes it several times slower. + ini-values: zend.assertions=1, error_reporting=-1, display_errors=On${{ inputs.coverage && inputs.coverage-driver == 'pcov' && format( ', pcov.directory={0}, "pcov.exclude=~/vendor/~"', github.workspace ) || '' }} extensions: gd, imagick, mysql, zip, pdo_sqlite, exif, fileinfo - coverage: ${{ inputs.coverage && 'xdebug' || 'none' }} + coverage: ${{ inputs.coverage && inputs.coverage-driver || 'none' }} tools: composer env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -236,23 +255,33 @@ jobs: # Spread the scenarios over parallel workers, each in its own throwaway containers. # Where Docker cannot be used, such as on Windows and macOS or when collecting code # coverage, wp-cli/wp-cli-tests falls back on its own. Older versions ignore these. - WP_CLI_TEST_PARALLEL: 4 + WP_CLI_TEST_PARALLEL: ${{ inputs.parallel }} + # The test framework spreads the scenarios over the shards by their expected cost, + # the same way on every runner, so that together they run every scenario once. + WP_CLI_TEST_SHARD: ${{ inputs.shard }} WP_CLI_TEST_DOCKER: auto run: | composer behat -- $BEHAT_ARGS || composer behat-rerun -- $BEHAT_ARGS - - name: Retrieve list of coverage files - id: coverage_files + # The calling workflow uploads the reports of all shards together, once all of them + # finished, so that Codecov receives them as one upload and never reports on a part. + - name: Name the code coverage report + id: coverage_artifact if: ${{ inputs.coverage }} - run: | - FILES=$(find "$GITHUB_WORKSPACE/build/logs" -path '*.*' | paste -s -d "," -) - echo "files=$FILES" >> $GITHUB_OUTPUT + env: + SHARD: ${{ inputs.shard || 'all' }} + PHP: ${{ inputs.php }} + DRIVER: ${{ inputs.coverage-driver }} + PARALLEL: ${{ inputs.parallel }} + run: echo "name=behat-coverage-${PHP}-${DRIVER}-p${PARALLEL}-${SHARD//\//-of-}" >> "$GITHUB_OUTPUT" - - name: Upload code coverage report + - name: Keep the code coverage report for the upload if: ${{ inputs.coverage }} - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - # Because somehow providing `directory: build/logs` doesn't work for these files - files: ${{ steps.coverage_files.outputs.files }} - flags: feature - token: ${{ secrets.CODECOV_TOKEN }} + name: ${{ steps.coverage_artifact.outputs.name }} + path: build/logs/*.xml + if-no-files-found: error + retention-days: 1 + # A re-run of a failed shard replaces the report of its previous attempt. + overwrite: true diff --git a/.github/workflows/reusable-testing.yml b/.github/workflows/reusable-testing.yml index 339f994..acb3f2d 100644 --- a/.github/workflows/reusable-testing.yml +++ b/.github/workflows/reusable-testing.yml @@ -18,6 +18,11 @@ on: type: boolean required: false default: true + coverage-shards: + description: 'Number of runners to split the Behat tests with code coverage across.' + type: number + required: false + default: 3 matrix: description: 'Additional matrix entries to include or exclude.' type: string @@ -44,6 +49,7 @@ jobs: outputs: unit: ${{ steps.unit.outputs.matrix }} functional: ${{ steps.functional.outputs.matrix }} + coverage: ${{ steps.functional.outputs.coverage }} steps: - name: Check out source code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -475,9 +481,10 @@ jobs: INPUTS_MINIMUM_PHP: ${{ inputs.minimum-php }} INPUTS_MINIMUM_WP: ${{ inputs.minimum-wp }} WITH_COVERAGE: ${{ inputs.with-coverage }} + COVERAGE_SHARDS: ${{ inputs.coverage-shards }} run: | if [[ $FILE_EXISTS == 'true' && $DOCS_ONLY != 'true' ]]; then - echo "matrix=$(jq -c \ + MATRIX=$(jq -c \ --argjson with_coverage_flag "${WITH_COVERAGE}" \ --arg minimum_php "${INPUTS_MINIMUM_PHP}" \ --arg minimum_wp "${INPUTS_MINIMUM_WP}" \ @@ -505,9 +512,28 @@ jobs: ).wp |= $minimum_wp ) ) - ' <<< "$BASE_MATRIX" )" >> "$GITHUB_OUTPUT" + ' <<< "$BASE_MATRIX" ) + + # Collecting code coverage makes the Behat tests several times slower, so these + # run as a job of their own, split across runners. An entry can set its own + # number of shards with "coverage_shards". + COVERAGE=$(jq -c --argjson shards "${COVERAGE_SHARDS}" ' + [ + .include[] | select(.coverage == true) | . as $entry | + ($entry.coverage_shards // $shards) as $total | + if $total > 1 then + range(1; $total + 1) | $entry + { shard: "\(.)/\($total)" } + else + $entry + { shard: "" } + end + ] + ' <<< "$MATRIX") + + echo "matrix=$(jq -c '.include |= map(select(.coverage != true)) | if (.include | length) > 0 then . else empty end' <<< "$MATRIX")" >> "$GITHUB_OUTPUT" + echo "coverage=$(jq -c 'if length > 0 then { include: . } else empty end' <<< "$COVERAGE")" >> "$GITHUB_OUTPUT" else echo "matrix=" >> "$GITHUB_OUTPUT" + echo "coverage=" >> "$GITHUB_OUTPUT" fi unit: @@ -547,3 +573,61 @@ jobs: object_cache: ${{ matrix.object_cache }} coverage: ${{ matrix.coverage == true }} os: ${{ matrix.os || '' }} + + functional-coverage: + needs: prepare + if: ${{ needs.prepare.outputs.coverage != '' }} + # Static on purpose, see above. + name: Behat coverage + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.coverage) }} + uses: ./.github/workflows/reusable-functional.yml + secrets: inherit + with: + php: ${{ matrix.php }} + wp: ${{ matrix.wp }} + dbtype: ${{ matrix.dbtype || 'mysql' }} + mysql: ${{ matrix.mysql || '' }} + object_cache: ${{ matrix.object_cache }} + coverage: true + coverage-driver: ${{ matrix.coverage_driver || 'pcov' }} + shard: ${{ matrix.shard }} + parallel: ${{ matrix.parallel || 4 }} + os: ${{ matrix.os || '' }} + + # The reports of all shards are uploaded together, so that Codecov receives the coverage + # of the Behat tests as one upload however many runners collected it, and never reports + # on a part of it. Only when every shard passed, as one that failed has no report. + coverage-upload: + needs: functional-coverage + if: ${{ needs.functional-coverage.result == 'success' }} + name: Upload Behat code coverage + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out source code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + + - name: Download the code coverage reports of all shards + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: behat-coverage-* + path: build/coverage + + - name: Retrieve list of coverage files + id: coverage_files + run: | + FILES=$(find "$GITHUB_WORKSPACE/build/coverage" -name '*.xml' | sort | paste -s -d "," -) + echo "Found $(tr ',' '\n' <<< "$FILES" | grep -c .) coverage reports." + echo "files=$FILES" >> "$GITHUB_OUTPUT" + + - name: Upload code coverage report + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + files: ${{ steps.coverage_files.outputs.files }} + flags: feature + token: ${{ secrets.CODECOV_TOKEN }} From 0df77727fa4535cb611d35801a957c0dbf27040e Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 04:23:42 +0000 Subject: [PATCH 2/3] Split the Behat coverage across 4 runners by default Co-Authored-By: Claude Opus 5.5 --- .github/workflows/reusable-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-testing.yml b/.github/workflows/reusable-testing.yml index acb3f2d..2252ac2 100644 --- a/.github/workflows/reusable-testing.yml +++ b/.github/workflows/reusable-testing.yml @@ -22,7 +22,7 @@ on: description: 'Number of runners to split the Behat tests with code coverage across.' type: number required: false - default: 3 + default: 4 matrix: description: 'Additional matrix entries to include or exclude.' type: string From 04d2d0dbc9b17d3d30bb59221244e2694c6d12d9 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 09:33:45 +0000 Subject: [PATCH 3/3] Size the coverage shards to the package, and cope without sharding support Every runner spends about a minute on its setup, so packages now get one coverage runner per 50 scenarios, up to the coverage-shards input. Versions of wp-cli/wp-cli-tests without sharding support ignore WP_CLI_TEST_SHARD and would run every scenario on every shard. With those, only the first shard runs the scenarios, so that each still runs once. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/reusable-functional.yml | 29 +++++++++++++++++++---- .github/workflows/reusable-testing.yml | 14 +++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-functional.yml b/.github/workflows/reusable-functional.yml index 84210ac..13ec789 100644 --- a/.github/workflows/reusable-functional.yml +++ b/.github/workflows/reusable-functional.yml @@ -233,11 +233,31 @@ jobs: if: ${{ inputs.dbtype != 'sqlite' && !startsWith(inputs.os, 'windows') && !startsWith(inputs.os, 'macos') }} run: sudo rm -f /usr/bin/mysql /usr/bin/mysqldump + # Versions of wp-cli/wp-cli-tests before the sharding support run every scenario on + # every shard. Then only the first shard runs them, so that they run once. + - name: Check whether wp-cli/wp-cli-tests can split the scenarios + id: shard + if: ${{ inputs.shard != '' }} + env: + SHARD: ${{ inputs.shard }} + run: | + RUNNER=vendor/wp-cli/wp-cli-tests/bin/run-behat-tests + [ -f "$RUNNER" ] || RUNNER=bin/run-behat-tests + if [ -f "$RUNNER" ] && grep -q WP_CLI_TEST_SHARD "$RUNNER"; then + echo "shard=${SHARD}" >> "$GITHUB_OUTPUT" + elif [ "${SHARD%%/*}" = 1 ]; then + echo "::warning::This version of wp-cli/wp-cli-tests cannot split the scenarios across runners, so shard ${SHARD} runs all of them." + else + echo "This version of wp-cli/wp-cli-tests cannot split the scenarios across runners, so shard 1 runs all of them and this one none." + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Prepare test database - if: ${{ inputs.dbtype != 'sqlite' }} + if: ${{ inputs.dbtype != 'sqlite' && steps.shard.outputs.skip != 'true' }} run: composer prepare-tests - name: Check Behat environment + if: ${{ steps.shard.outputs.skip != 'true' }} env: WP_VERSION: '${{ inputs.wp }}' WP_CLI_TEST_DBTYPE: ${{ inputs.dbtype || 'mysql' }} @@ -246,6 +266,7 @@ jobs: run: composer behat - name: Run Behat + if: ${{ steps.shard.outputs.skip != 'true' }} env: WP_VERSION: '${{ inputs.wp }}' WP_CLI_TEST_DBTYPE: ${{ inputs.dbtype || 'mysql' }} @@ -258,7 +279,7 @@ jobs: WP_CLI_TEST_PARALLEL: ${{ inputs.parallel }} # The test framework spreads the scenarios over the shards by their expected cost, # the same way on every runner, so that together they run every scenario once. - WP_CLI_TEST_SHARD: ${{ inputs.shard }} + WP_CLI_TEST_SHARD: ${{ steps.shard.outputs.shard }} WP_CLI_TEST_DOCKER: auto run: | composer behat -- $BEHAT_ARGS || composer behat-rerun -- $BEHAT_ARGS @@ -267,7 +288,7 @@ jobs: # finished, so that Codecov receives them as one upload and never reports on a part. - name: Name the code coverage report id: coverage_artifact - if: ${{ inputs.coverage }} + if: ${{ inputs.coverage && steps.shard.outputs.skip != 'true' }} env: SHARD: ${{ inputs.shard || 'all' }} PHP: ${{ inputs.php }} @@ -276,7 +297,7 @@ jobs: run: echo "name=behat-coverage-${PHP}-${DRIVER}-p${PARALLEL}-${SHARD//\//-of-}" >> "$GITHUB_OUTPUT" - name: Keep the code coverage report for the upload - if: ${{ inputs.coverage }} + if: ${{ inputs.coverage && steps.shard.outputs.skip != 'true' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ steps.coverage_artifact.outputs.name }} diff --git a/.github/workflows/reusable-testing.yml b/.github/workflows/reusable-testing.yml index 2252ac2..0f9fc31 100644 --- a/.github/workflows/reusable-testing.yml +++ b/.github/workflows/reusable-testing.yml @@ -19,7 +19,7 @@ on: required: false default: true coverage-shards: - description: 'Number of runners to split the Behat tests with code coverage across.' + description: 'Maximum number of runners to split the Behat tests with code coverage across. Packages get one runner per 50 scenarios, up to this number.' type: number required: false default: 4 @@ -482,6 +482,7 @@ jobs: INPUTS_MINIMUM_WP: ${{ inputs.minimum-wp }} WITH_COVERAGE: ${{ inputs.with-coverage }} COVERAGE_SHARDS: ${{ inputs.coverage-shards }} + SCENARIOS_PER_SHARD: 50 run: | if [[ $FILE_EXISTS == 'true' && $DOCS_ONLY != 'true' ]]; then MATRIX=$(jq -c \ @@ -515,9 +516,14 @@ jobs: ' <<< "$BASE_MATRIX" ) # Collecting code coverage makes the Behat tests several times slower, so these - # run as a job of their own, split across runners. An entry can set its own - # number of shards with "coverage_shards". - COVERAGE=$(jq -c --argjson shards "${COVERAGE_SHARDS}" ' + # run as a job of their own, split across runners. Every runner spends about a + # minute on its setup, so small packages use fewer of them. An entry can set its + # own number of shards with "coverage_shards". + SCENARIOS=$(find features -name '*.feature' -exec cat {} + 2>/dev/null | grep -cE '^[[:space:]]*Scenario( Outline)?:' || true) + SHARDS=$(( (SCENARIOS + SCENARIOS_PER_SHARD - 1) / SCENARIOS_PER_SHARD )) + SHARDS=$(( SHARDS < 1 ? 1 : SHARDS > COVERAGE_SHARDS ? COVERAGE_SHARDS : SHARDS )) + echo "Splitting the Behat tests with code coverage across ${SHARDS} runners, for ${SCENARIOS} scenarios." + COVERAGE=$(jq -c --argjson shards "${SHARDS}" ' [ .include[] | select(.coverage == true) | . as $entry | ($entry.coverage_shards // $shards) as $total |