diff --git a/.github/workflows/reusable-functional.yml b/.github/workflows/reusable-functional.yml index ee3dfa3..13ec789 100644 --- a/.github/workflows/reusable-functional.yml +++ b/.github/workflows/reusable-functional.yml @@ -24,6 +24,21 @@ on: type: boolean required: false default: false + coverage-driver: + description: 'Code coverage driver, either `pcov` or `xdebug`.' + type: string + required: false + default: 'pcov' + shard: + description: 'Share of the scenarios to run, as `/`, for example `2/3`. Empty to run all of them.' + type: string + required: false + default: '' + parallel: + description: 'Number of parallel workers on the runner.' + type: number + required: false + default: 4 os: description: 'Runner to use. Defaults to ubuntu-22.04, or the RUNNERS_NAME repository variable when set.' type: string @@ -40,7 +55,7 @@ jobs: # anything omitted here is not shown anywhere. The database version is spelled # out because "MySQL" alone rendered the mysql-8.0 and mysql-8.4 legs # identically. - name: Behat | PHP ${{ inputs.php }} | WP ${{ inputs.wp }} | ${{ inputs.dbtype == 'sqlite' && 'SQLite' || inputs.mysql || 'MySQL' }}${{ inputs.object_cache == 'sqlite' && ' (Obj Cache)' || '' }}${{ inputs.coverage && ' (with coverage)' || '' }}${{ startsWith( inputs.os, 'windows' ) && ' (Windows)' || '' }}${{ startsWith( inputs.os, 'macos' ) && ' (macOS)' || '' }} + name: Behat | PHP ${{ inputs.php }} | WP ${{ inputs.wp }} | ${{ inputs.dbtype == 'sqlite' && 'SQLite' || inputs.mysql || 'MySQL' }}${{ inputs.object_cache == 'sqlite' && ' (Obj Cache)' || '' }}${{ inputs.coverage && format( ' (with coverage{0})', inputs.shard && format( ', {0}', inputs.shard ) || '' ) || '' }}${{ startsWith( inputs.os, 'windows' ) && ' (Windows)' || '' }}${{ startsWith( inputs.os, 'macos' ) && ' (macOS)' || '' }} # Repositories with a heavy Behat suite can point the default Linux legs at a # larger runner by setting the `RUNNERS_NAME` repository variable, without # having to fork this workflow. Explicit macOS/Windows legs are unaffected. @@ -166,9 +181,13 @@ jobs: uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 with: php-version: '${{ inputs.php }}' - ini-values: zend.assertions=1, error_reporting=-1, display_errors=On + # PCOV only instruments the files below pcov.directory. Left unset, it would pick the + # working directory of every `wp` process the scenarios run, which is a temporary + # WordPress install, and collect nothing. Xdebug instead instruments every file it + # loads, all of WordPress included, which makes it several times slower. + ini-values: zend.assertions=1, error_reporting=-1, display_errors=On${{ inputs.coverage && inputs.coverage-driver == 'pcov' && format( ', pcov.directory={0}, "pcov.exclude=~/vendor/~"', github.workspace ) || '' }} extensions: gd, imagick, mysql, zip, pdo_sqlite, exif, fileinfo - coverage: ${{ inputs.coverage && 'xdebug' || 'none' }} + coverage: ${{ inputs.coverage && inputs.coverage-driver || 'none' }} tools: composer env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -214,11 +233,31 @@ jobs: if: ${{ inputs.dbtype != 'sqlite' && !startsWith(inputs.os, 'windows') && !startsWith(inputs.os, 'macos') }} run: sudo rm -f /usr/bin/mysql /usr/bin/mysqldump + # Versions of wp-cli/wp-cli-tests before the sharding support run every scenario on + # every shard. Then only the first shard runs them, so that they run once. + - name: Check whether wp-cli/wp-cli-tests can split the scenarios + id: shard + if: ${{ inputs.shard != '' }} + env: + SHARD: ${{ inputs.shard }} + run: | + RUNNER=vendor/wp-cli/wp-cli-tests/bin/run-behat-tests + [ -f "$RUNNER" ] || RUNNER=bin/run-behat-tests + if [ -f "$RUNNER" ] && grep -q WP_CLI_TEST_SHARD "$RUNNER"; then + echo "shard=${SHARD}" >> "$GITHUB_OUTPUT" + elif [ "${SHARD%%/*}" = 1 ]; then + echo "::warning::This version of wp-cli/wp-cli-tests cannot split the scenarios across runners, so shard ${SHARD} runs all of them." + else + echo "This version of wp-cli/wp-cli-tests cannot split the scenarios across runners, so shard 1 runs all of them and this one none." + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Prepare test database - if: ${{ inputs.dbtype != 'sqlite' }} + if: ${{ inputs.dbtype != 'sqlite' && steps.shard.outputs.skip != 'true' }} run: composer prepare-tests - name: Check Behat environment + if: ${{ steps.shard.outputs.skip != 'true' }} env: WP_VERSION: '${{ inputs.wp }}' WP_CLI_TEST_DBTYPE: ${{ inputs.dbtype || 'mysql' }} @@ -227,6 +266,7 @@ jobs: run: composer behat - name: Run Behat + if: ${{ steps.shard.outputs.skip != 'true' }} env: WP_VERSION: '${{ inputs.wp }}' WP_CLI_TEST_DBTYPE: ${{ inputs.dbtype || 'mysql' }} @@ -236,23 +276,33 @@ jobs: # Spread the scenarios over parallel workers, each in its own throwaway containers. # Where Docker cannot be used, such as on Windows and macOS or when collecting code # coverage, wp-cli/wp-cli-tests falls back on its own. Older versions ignore these. - WP_CLI_TEST_PARALLEL: 4 + WP_CLI_TEST_PARALLEL: ${{ inputs.parallel }} + # The test framework spreads the scenarios over the shards by their expected cost, + # the same way on every runner, so that together they run every scenario once. + WP_CLI_TEST_SHARD: ${{ steps.shard.outputs.shard }} WP_CLI_TEST_DOCKER: auto run: | composer behat -- $BEHAT_ARGS || composer behat-rerun -- $BEHAT_ARGS - - name: Retrieve list of coverage files - id: coverage_files - if: ${{ inputs.coverage }} - run: | - FILES=$(find "$GITHUB_WORKSPACE/build/logs" -path '*.*' | paste -s -d "," -) - echo "files=$FILES" >> $GITHUB_OUTPUT + # The calling workflow uploads the reports of all shards together, once all of them + # finished, so that Codecov receives them as one upload and never reports on a part. + - name: Name the code coverage report + id: coverage_artifact + if: ${{ inputs.coverage && steps.shard.outputs.skip != 'true' }} + env: + SHARD: ${{ inputs.shard || 'all' }} + PHP: ${{ inputs.php }} + DRIVER: ${{ inputs.coverage-driver }} + PARALLEL: ${{ inputs.parallel }} + run: echo "name=behat-coverage-${PHP}-${DRIVER}-p${PARALLEL}-${SHARD//\//-of-}" >> "$GITHUB_OUTPUT" - - name: Upload code coverage report - if: ${{ inputs.coverage }} - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + - name: Keep the code coverage report for the upload + if: ${{ inputs.coverage && steps.shard.outputs.skip != 'true' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - # Because somehow providing `directory: build/logs` doesn't work for these files - files: ${{ steps.coverage_files.outputs.files }} - flags: feature - token: ${{ secrets.CODECOV_TOKEN }} + name: ${{ steps.coverage_artifact.outputs.name }} + path: build/logs/*.xml + if-no-files-found: error + retention-days: 1 + # A re-run of a failed shard replaces the report of its previous attempt. + overwrite: true diff --git a/.github/workflows/reusable-testing.yml b/.github/workflows/reusable-testing.yml index 339f994..0f9fc31 100644 --- a/.github/workflows/reusable-testing.yml +++ b/.github/workflows/reusable-testing.yml @@ -18,6 +18,11 @@ on: type: boolean required: false default: true + coverage-shards: + description: 'Maximum number of runners to split the Behat tests with code coverage across. Packages get one runner per 50 scenarios, up to this number.' + type: number + required: false + default: 4 matrix: description: 'Additional matrix entries to include or exclude.' type: string @@ -44,6 +49,7 @@ jobs: outputs: unit: ${{ steps.unit.outputs.matrix }} functional: ${{ steps.functional.outputs.matrix }} + coverage: ${{ steps.functional.outputs.coverage }} steps: - name: Check out source code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -475,9 +481,11 @@ jobs: INPUTS_MINIMUM_PHP: ${{ inputs.minimum-php }} INPUTS_MINIMUM_WP: ${{ inputs.minimum-wp }} WITH_COVERAGE: ${{ inputs.with-coverage }} + COVERAGE_SHARDS: ${{ inputs.coverage-shards }} + SCENARIOS_PER_SHARD: 50 run: | if [[ $FILE_EXISTS == 'true' && $DOCS_ONLY != 'true' ]]; then - echo "matrix=$(jq -c \ + MATRIX=$(jq -c \ --argjson with_coverage_flag "${WITH_COVERAGE}" \ --arg minimum_php "${INPUTS_MINIMUM_PHP}" \ --arg minimum_wp "${INPUTS_MINIMUM_WP}" \ @@ -505,9 +513,33 @@ jobs: ).wp |= $minimum_wp ) ) - ' <<< "$BASE_MATRIX" )" >> "$GITHUB_OUTPUT" + ' <<< "$BASE_MATRIX" ) + + # Collecting code coverage makes the Behat tests several times slower, so these + # run as a job of their own, split across runners. Every runner spends about a + # minute on its setup, so small packages use fewer of them. An entry can set its + # own number of shards with "coverage_shards". + SCENARIOS=$(find features -name '*.feature' -exec cat {} + 2>/dev/null | grep -cE '^[[:space:]]*Scenario( Outline)?:' || true) + SHARDS=$(( (SCENARIOS + SCENARIOS_PER_SHARD - 1) / SCENARIOS_PER_SHARD )) + SHARDS=$(( SHARDS < 1 ? 1 : SHARDS > COVERAGE_SHARDS ? COVERAGE_SHARDS : SHARDS )) + echo "Splitting the Behat tests with code coverage across ${SHARDS} runners, for ${SCENARIOS} scenarios." + COVERAGE=$(jq -c --argjson shards "${SHARDS}" ' + [ + .include[] | select(.coverage == true) | . as $entry | + ($entry.coverage_shards // $shards) as $total | + if $total > 1 then + range(1; $total + 1) | $entry + { shard: "\(.)/\($total)" } + else + $entry + { shard: "" } + end + ] + ' <<< "$MATRIX") + + echo "matrix=$(jq -c '.include |= map(select(.coverage != true)) | if (.include | length) > 0 then . else empty end' <<< "$MATRIX")" >> "$GITHUB_OUTPUT" + echo "coverage=$(jq -c 'if length > 0 then { include: . } else empty end' <<< "$COVERAGE")" >> "$GITHUB_OUTPUT" else echo "matrix=" >> "$GITHUB_OUTPUT" + echo "coverage=" >> "$GITHUB_OUTPUT" fi unit: @@ -547,3 +579,61 @@ jobs: object_cache: ${{ matrix.object_cache }} coverage: ${{ matrix.coverage == true }} os: ${{ matrix.os || '' }} + + functional-coverage: + needs: prepare + if: ${{ needs.prepare.outputs.coverage != '' }} + # Static on purpose, see above. + name: Behat coverage + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.coverage) }} + uses: ./.github/workflows/reusable-functional.yml + secrets: inherit + with: + php: ${{ matrix.php }} + wp: ${{ matrix.wp }} + dbtype: ${{ matrix.dbtype || 'mysql' }} + mysql: ${{ matrix.mysql || '' }} + object_cache: ${{ matrix.object_cache }} + coverage: true + coverage-driver: ${{ matrix.coverage_driver || 'pcov' }} + shard: ${{ matrix.shard }} + parallel: ${{ matrix.parallel || 4 }} + os: ${{ matrix.os || '' }} + + # The reports of all shards are uploaded together, so that Codecov receives the coverage + # of the Behat tests as one upload however many runners collected it, and never reports + # on a part of it. Only when every shard passed, as one that failed has no report. + coverage-upload: + needs: functional-coverage + if: ${{ needs.functional-coverage.result == 'success' }} + name: Upload Behat code coverage + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out source code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + + - name: Download the code coverage reports of all shards + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: behat-coverage-* + path: build/coverage + + - name: Retrieve list of coverage files + id: coverage_files + run: | + FILES=$(find "$GITHUB_WORKSPACE/build/coverage" -name '*.xml' | sort | paste -s -d "," -) + echo "Found $(tr ',' '\n' <<< "$FILES" | grep -c .) coverage reports." + echo "files=$FILES" >> "$GITHUB_OUTPUT" + + - name: Upload code coverage report + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + files: ${{ steps.coverage_files.outputs.files }} + flags: feature + token: ${{ secrets.CODECOV_TOKEN }}