From 6870d1f022f078dd57c4c8a477c87bb2865bfe47 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 01:34:05 -0700 Subject: [PATCH 01/27] Build/Test Tools: Add plugin compatibility testing workflow. Core CI covers core itself, but nothing checks that a new version of WordPress can still boot with popular plugins active. When a plugin's assumptions about core stop holding the result is a fatal error on every request, which is a white screen for real sites and is only discovered after release. Add a workflow that fetches the most popular plugins from the WordPress.org API at run time, then installs and activates each one on its own against the version of WordPress under test. A fatal is caught whether it happens on activation, while WP-CLI loads WordPress, on a front end or login request, or in the debug log, so a white screen with error display turned off is still detected. Failures are reported per plugin in the workflow summary and one broken plugin never stops the rest of the shard from being tested. Plugins that cannot be downloaded are reported as skipped rather than failed so that a network flake does not turn the run red. The run is manual or weekly rather than part of every commit, since a third party plugin breaking should be a signal to release leads, not a red check on unrelated work. --- .github/workflows/plugin-compatibility.yml | 205 ++++++++++++ .../reusable-plugin-compatibility.yml | 316 ++++++++++++++++++ 2 files changed, 521 insertions(+) create mode 100644 .github/workflows/plugin-compatibility.yml create mode 100644 .github/workflows/reusable-plugin-compatibility.yml diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..dc8d2f709c576 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,205 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each +# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + type: string + default: 'nightly' + plugin-count: + description: 'How many of the most popular plugins to test.' + type: string + default: '100' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Fetch the most popular plugins + id: plugin-shards + env: + # A pull request or scheduled run has no inputs, so fall back to the default count. + PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} + SHARD_COUNT: '5' + run: | + set -euo pipefail + + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer.\n' + exit 1 + fi + + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins.json" + + SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + needs: [ slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..8147b6b5ec4b1 --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,316 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' + required: true + type: 'string' + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Sets up PHP. + # - Downloads the specified version of WordPress. + # - Creates a `wp-config.php` file with debugging and error logging enabled. + # - Installs WordPress. + # - Starts the PHP built-in web server so HTTP requests can be made against the site. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + timeout-minutes: 30 + + services: + database: + image: ${{ inputs.db-type }}:${{ inputs.db-version }} + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + - name: Download WordPress ${{ inputs.wp-version }} + run: wp core download --version="${WP_VERSION}" + env: + WP_VERSION: ${{ inputs.wp-version }} + + - name: Create wp-config.php file + run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" + env: + DB_PORT: ${{ job.services.database.ports['3306'] }} + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by + # recovery mode, which would also deactivate the plugin mid-test. + - name: Enable debugging and error logging + run: | + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + + - name: Install WordPress + run: | + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email + env: + SITE_URL: http://127.0.0.1:8889 + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + - name: Start the PHP built-in web server + run: | + set -uo pipefail + + nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & + + # Wait for the server to start answering before any plugin is installed. + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then + printf 'The PHP built-in server is ready.\n' + exit 0 + fi + sleep 1 + done + + printf 'The PHP built-in server did not start.\n' + cat "${RUNNER_TEMP}/php-server.log" + exit 1 + + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + PHP_VERSION: ${{ inputs.php-version }} + SITE_URL: http://127.0.0.1:8889 + WP_VERSION: ${{ inputs.wp-version }} + run: | + # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining + # plugins in the shard from being tested. + set -uo pipefail + + RESULTS="${RUNNER_TEMP}/plugin-results.tsv" + RESPONSE_BODY="${RUNNER_TEMP}/response.html" + : > "${RESULTS}" + + # record + # + # Appends one tab separated row to the results file. The reason is flattened so that it cannot + # break the markdown table that is generated from these rows later on. + record() { + SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + } + + # check_url + # + # Requests a path on the test site and prints a reason to stdout when the response looks broken. + # Prints nothing when the request looks healthy. + check_url() { + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" + HTTP_CODE="${HTTP_CODE:-000}" + + if [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete' "${1}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors + # back on for itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi + } + + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # cleanup_plugin + # + # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # command here is allowed to fail and the plugin directory is removed directly as a fallback. + # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. + cleanup_plugin() { + wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + + if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${2}" + fi + + rm -rf "wp-content/plugins/${1}" + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true + } + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + printf '::group::%s\n' "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + PLUGIN_DIR="" + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f wp-content/debug.log + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, + # since it generally means a network flake or a plugin that is no longer in the directory. + if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then + record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" + cleanup_plugin "${SLUG}" "" + printf '::endgroup::\n' + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. + if ! wp plugin activate "${SLUG}"; then + STATUS="FAIL" + REASON="The plugin could not be activated" + fi + + # Step 3: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + EVAL_EXIT_CODE=$? + + if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + STATUS="FAIL" + REASON="WordPress could not be loaded by WP-CLI with the plugin active" + fi + fi + + # Step 4: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then + grep 'PHP Fatal' wp-content/debug.log + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" + fi + + # Step 6: record the outcome and put the site back the way it was found. + record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" + cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + printf '::endgroup::\n' + done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) + + PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" + FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" + SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" + + { + printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" + printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" + printf '| Plugin | Version | Result | Details |\n' + printf '| --- | --- | --- | --- |\n' + + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + case "${ROW_STATUS}" in + PASS ) ICON=':white_check_mark:' ;; + FAIL ) ICON=':x:' ;; + * ) ICON=':warning:' ;; + esac + + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + done < "${RESULTS}" + + printf '\n' + } >> "${GITHUB_STEP_SUMMARY}" + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + exit 1 + fi + + printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" + + - name: Show the web server log + if: ${{ failure() }} + run: cat "${RUNNER_TEMP}/php-server.log" From 14340c7a4bde129331a33bfbbb54bd284f8c1925 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 02/27] Build/Test Tools: Correct plugin compatibility failure classification. The first run of this workflow reported four failures that were not fatals. Core refuses to activate a plugin whose `Requires Plugins` dependency is missing, which every WooCommerce extension hits when plugins are tested one at a time, and WP-CLI exits non-zero when a plugin redirects while loading. Both are correct behaviour, so record them as skipped and reserve a failure for an actual fatal. A front end request also reported the nonsense status "200000", because the curl fallback appended to output curl had already written. Capture the exit code separately so a stalled transfer is reported as what it is. The stall itself came from WordPress spawning WP-Cron as a loopback request that the single threaded built-in server could not answer while still serving the request that spawned it. Disable WP-Cron and give the server workers so plugin loopback requests cannot deadlock it. Ignore the zizmor unpinned image finding on the database service, which cannot be pinned to a digest while the version is an input. --- .../reusable-plugin-compatibility.yml | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 8147b6b5ec4b1..4ca645f3c7aa9 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -64,7 +64,10 @@ jobs: services: database: - image: ${{ inputs.db-type }}:${{ inputs.db-version }} + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] ports: - 3306 options: >- @@ -100,12 +103,18 @@ jobs: # # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by # recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned + # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is + # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. - name: Enable debugging and error logging run: | wp config set WP_DEBUG true --raw wp config set WP_DEBUG_LOG true --raw wp config set WP_DEBUG_DISPLAY false --raw wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw - name: Install WordPress run: | @@ -116,7 +125,13 @@ jobs: # The site needs to answer real requests so that fatals which only happen on a front end or admin page # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. - name: Start the PHP built-in web server + env: + PHP_CLI_SERVER_WORKERS: '4' run: | set -uo pipefail @@ -164,11 +179,14 @@ jobs: # Requests a path on the test site and prints a reason to stdout when the response looks broken. # Prints nothing when the request looks healthy. check_url() { - HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" - HTTP_CODE="${HTTP_CODE:-000}" - - if [ "${HTTP_CODE}" = "000" ]; then - printf 'The request to %s did not complete' "${1}" + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense + # status like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" return fi @@ -177,6 +195,11 @@ jobs: return fi + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors # back on for itself. if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then @@ -232,20 +255,46 @@ jobs: PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - if ! wp plugin activate "${SLUG}"; then - STATUS="FAIL" - REASON="The plugin could not be activated" + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met, most often a + # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means + # every WooCommerce extension lands here. That is core working as designed rather than a + # fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac fi # Step 3: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" - EVAL_EXIT_CODE=$? + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true - if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then printf '%s\n' "${EVAL_OUTPUT}" - STATUS="FAIL" - REASON="WordPress could not be loaded by WP-CLI with the plugin active" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac fi fi From 915017705dcd307e748b4e62a30762410f3d789c Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 03/27] Build/Test Tools: Support any plugin count in compatibility testing. The WordPress.org API caps `per_page` at 250 and quietly returns 250 for anything larger, so asking for more than that silently tested fewer plugins than requested. Page through the API instead and trim to the requested count, de-duplicating across pages because popularity ordering can shift between two requests. Reject a count above 1000 with a clear message rather than truncating without saying so. Size the shards to the plugin count rather than always splitting into five, so a run of 10 does not spin up five near empty jobs and a run of 250 is not squeezed into the same five. Point pull request and push runs at the latest stable release with a small count. Those runs exist to check that this workflow still works, and a genuine ecosystem fatal against nightly should not sit as a red check on every later change to these files. --- .github/workflows/plugin-compatibility.yml | 128 ++++++++++++++++----- 1 file changed, 100 insertions(+), 28 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index dc8d2f709c576..3ae64ce5abeaf 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -7,8 +7,13 @@ # every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so # that it can be found while there is still time to fix core or notify the plugin author. # -# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each -# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. # # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. @@ -36,11 +41,11 @@ on: workflow_dispatch: inputs: wp-version: - description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' type: string default: 'nightly' plugin-count: - description: 'How many of the most popular plugins to test.' + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' type: string default: '100' @@ -78,36 +83,84 @@ jobs: - name: Fetch the most popular plugins id: plugin-shards env: - # A pull request or scheduled run has no inputs, so fall back to the default count. - PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} - SHARD_COUNT: '5' + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' run: | set -euo pipefail # Guard against a non-numeric value being passed to the API. if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then - printf 'The plugin-count input must be a positive integer.\n' + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" exit 1 fi - # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. - # The unneeded response fields are turned off to keep the payload small. - curl -sS --fail --retry 3 --retry-delay 5 \ - --get 'https://api.wordpress.org/plugins/info/1.2/' \ - --data-urlencode 'action=query_plugins' \ - --data-urlencode 'request[browse]=popular' \ - --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ - --data-urlencode 'request[fields][short_description]=0' \ - --data-urlencode 'request[fields][sections]=0' \ - --data-urlencode 'request[fields][icons]=0' \ - --data-urlencode 'request[fields][banners]=0' \ - --data-urlencode 'request[fields][ratings]=0' \ - --data-urlencode 'request[fields][tags]=0' \ - --data-urlencode 'request[fields][compatibility]=0' \ - --data-urlencode 'request[fields][screenshots]=0' \ - -o "${RUNNER_TEMP}/plugins.json" - - SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" if [ "${TOTAL}" -lt 1 ]; then @@ -115,6 +168,19 @@ jobs: exit 1 fi + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer # plugins were requested than there are shards. Each shard's slugs are passed to the reusable # workflow as a JSON string. @@ -135,8 +201,14 @@ jobs: printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. plugin-compatibility-tests: - name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} uses: ./.github/workflows/reusable-plugin-compatibility.yml permissions: contents: read @@ -148,7 +220,7 @@ jobs: shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} with: os: 'ubuntu-24.04' - wp-version: ${{ inputs.wp-version || 'nightly' }} + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} php-version: '8.3' plugin-slugs: ${{ matrix.shard.slugs }} db-type: 'mysql' From cf18ee6356faabb66a7705db9daf7d300bb8d960 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:05:30 -0700 Subject: [PATCH 04/27] Build/Test Tools: Fix shard count wording in plugin compatibility log. A run that builds a single shard logged "across 1 shards". --- .github/workflows/plugin-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 3ae64ce5abeaf..b2a9c12d2d6a6 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -195,7 +195,7 @@ jobs: | map( { index: .index, slugs: ( .slugs | @json ) } ) ' )" - printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" From 67005f527ba98d30586aa98c1ad89f25fb1867f3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 25 Aug 2026 11:58:18 -0700 Subject: [PATCH 05/27] Build/Test Tools: Install plugin dependencies before compatibility testing. Testing each plugin on its own meant core refused to activate anything with a `Requires Plugins` header, so every WooCommerce extension was recorded as skipped without ever being loaded. Those extensions are a large slice of the most popular plugins, which left a gap in exactly the part of the ecosystem the workflow exists to watch. Read the `Requires Plugins` header with WordPress' own parser, install and activate what it names, and only then activate the plugin under test. Chains are followed a level at a time so a dependency declaring its own is installed too, and anything already present is left alone, which terminates a circular declaration. Dependencies mean the plugin under test is no longer alone on the site, so the front page and login screen are checked with only the dependencies active first. A site already broken at that point is the dependency's doing, and the plugin is skipped rather than blamed for it. The debug log is cleared at the same point for the same reason. The results table gains a column naming what else was active so a failure can be read in context. Installing and activating a dependency the size of WooCommerce is not quick, so the job timeout goes from 30 to 45 minutes. --- .../reusable-plugin-compatibility.yml | 251 ++++++++++++++---- 1 file changed, 200 insertions(+), 51 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 4ca645f3c7aa9..d32d2156b9b16 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -4,6 +4,11 @@ # # Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then # removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. ## name: Plugin Compatibility Tests @@ -53,14 +58,17 @@ jobs: # - Creates a `wp-config.php` file with debugging and error logging enabled. # - Installs WordPress. # - Starts the PHP built-in web server so HTTP requests can be made against the site. - # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time, together with + # anything it declares in `Requires Plugins`. # - Writes a results table to the workflow summary and fails the job if any plugin fataled. plugin-compatibility-tests: name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} permissions: contents: read runs-on: ${{ inputs.os }} - timeout-minutes: 30 + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 services: database: @@ -165,13 +173,24 @@ jobs: RESPONSE_BODY="${RUNNER_TEMP}/response.html" : > "${RESULTS}" - # record + # How many levels of `Requires Plugins` to follow. Chains in the directory are one level + # deep in practice, so the bound is only here to stop a circular declaration from looping + # forever. + MAX_DEPENDENCY_DEPTH=3 + + # Everything installed for the plugin currently under test, that plugin included. The + # directories are tracked alongside the slugs because a plugin does not always unpack into + # a directory named after its slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # record # # Appends one tab separated row to the results file. The reason is flattened so that it cannot # break the markdown table that is generated from these rows later on. record() { - SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" - printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" } # check_url @@ -210,20 +229,70 @@ jobs: # The plugins directory itself must never be removed, only directories inside it. PLUGINS_ROOT="$( wp plugin path )" - # cleanup_plugin + # install_plugin # - # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # Installs a plugin from WordPress.org and remembers it so that it is removed again once the + # plugin under test has been checked. The slug is remembered before the download is attempted + # so that a partial download is still cleaned up. Returns non zero when the download failed. + install_plugin() { + INSTALLED_SLUGS+=( "${1}" ) + INSTALLED_DIRS+=( "" ) + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[-1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + } + + # plugin_dependencies + # + # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' + # own header parser is used so that the same rules apply here as when core decides whether a + # plugin's dependencies are met. + plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + # shellcheck disable=SC2016 + PLUGIN_FILE="${PLUGIN_FILE}" wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null + } + + # cleanup_plugins + # + # Returns the site to a clean slate by removing everything installed for the current plugin, + # its dependencies included. A plugin that fatals can take WP-CLI down with it, so every # command here is allowed to fail and the plugin directory is removed directly as a fallback. # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. - cleanup_plugin() { - wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + cleanup_plugins() { + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${2}" - fi + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - rm -rf "wp-content/plugins/${1}" + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "wp-content/plugins/${CLEANUP_SLUG}" + done + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true @@ -237,46 +306,119 @@ jobs: STATUS="PASS" REASON="-" VERSION="unknown" - PLUGIN_DIR="" + DEPENDENCIES=() # Start every plugin with an empty log so that anything found in it belongs to this plugin. rm -f wp-content/debug.log # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, # since it generally means a network flake or a plugin that is no longer in the directory. - if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then - record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" - cleanup_plugin "${SLUG}" "" + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins printf '::endgroup::\n' continue fi VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" - PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" - - # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - ACTIVATE_EXIT_CODE=0 - ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? - printf '%s\n' "${ACTIVATE_OUTPUT}" - - if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then - case "${ACTIVATE_OUTPUT}" in - # Core refuses to activate a plugin whose declared requirements are not met, most often a - # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means - # every WooCommerce extension lands here. That is core working as designed rather than a - # fatal, so it is recorded as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a + # plugin whose dependencies are missing, so without this every WooCommerce extension - a large + # slice of the most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is + # installed too. Anything already installed is left alone, which covers a plugin named twice + # in the same tree as well as a declaration pointing back at the plugin under test. + mapfile -t PENDING < <( plugin_dependencies "${SLUG}" ) + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then STATUS="SKIPPED" - REASON="Core declined to activate the plugin because its declared requirements are not met" - ;; - * ) - STATUS="FAIL" - REASON="The plugin could not be activated" - ;; - esac + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES+=( "${DEPENDENCY}" ) + mapfile -t -O "${#NEXT[@]}" NEXT < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( "${NEXT[@]}" ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they + # were discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so + # the baseline is checked before it is activated. A site that is already broken says something + # about the dependency rather than about the plugin being tested, and blaming the plugin for + # it would be the masking this workflow is built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. + rm -f wp-content/debug.log fi - # Step 3: boot all of core plus the active plugin in a CLI context. + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. Dependencies + # are installed above, so what is left here is a plugin asking for a version of PHP or + # WordPress this job is not running, or for a dependency that is not on WordPress.org. + # That is core working as designed rather than a fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true @@ -298,7 +440,7 @@ jobs: fi fi - # Step 4: request the front page and the login screen through the PHP built-in server. + # Step 7: request the front page and the login screen through the PHP built-in server. if [ "${STATUS}" = "PASS" ]; then for URL_PATH in "/" "/wp-login.php"; do HTTP_REASON="$( check_url "${URL_PATH}" )" @@ -311,7 +453,7 @@ jobs: done fi - # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown # hook, so the debug log is checked separately. if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then grep 'PHP Fatal' wp-content/debug.log @@ -319,9 +461,16 @@ jobs: REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" fi - # Step 6: record the outcome and put the site back the way it was found. - record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" - cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins printf '%s: %s\n' "${SLUG}" "${STATUS}" printf '::endgroup::\n' @@ -334,18 +483,18 @@ jobs: { printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" - printf '| Plugin | Version | Result | Details |\n' - printf '| --- | --- | --- | --- |\n' + printf '| Plugin | Version | Result | Also active | Details |\n' + printf '| --- | --- | --- | --- | --- |\n' - while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_DEPENDENCIES ROW_REASON; do case "${ROW_STATUS}" in PASS ) ICON=':white_check_mark:' ;; FAIL ) ICON=':x:' ;; * ) ICON=':warning:' ;; esac - printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ - "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_DEPENDENCIES}" "${ROW_REASON}" done < "${RESULTS}" printf '\n' @@ -354,7 +503,7 @@ jobs: # Plugins that could not be downloaded are reported but do not fail the run. if [ "${FAIL_COUNT}" -gt 0 ]; then printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" - awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" exit 1 fi From 06f801d8d660f6df80e409aa45477487a9d49a61 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 31 Aug 2026 16:22:12 -0700 Subject: [PATCH 06/27] Build/Test Tools: Allow plugin compatibility runs to name plugins. Trying this workflow out meant taking whatever the popularity query returned, so anyone wanting to check one plugin against a release candidate, or to reproduce a failure from an earlier run, had to sit through a full run and read past everything else in the shard. Reviewers could not run it at all: the repository guard skips both jobs everywhere but this repository, and dispatching it here needs write access. Add an optional `plugin-slugs` input that replaces the directory query with a list written by hand. Commas and whitespace both separate slugs so a list can be pasted in however it was written down, and a value outside the character set the directory uses stops the run immediately rather than being reported as an impossible download several minutes later. The list goes through the same sharding as a fetched one, so naming forty plugins still splits across two jobs. Let a dispatched run through the repository guard wherever it is triggered, which is what makes a fork useful for trying this out. Two runs dispatched against the same version of WordPress used to land in the same concurrency group and cancel each other, which is wrong once the plugins being tested can differ between them. Dispatched runs are now grouped by run id. --- .github/workflows/plugin-compatibility.yml | 158 +++++++++++++-------- 1 file changed, 102 insertions(+), 56 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index b2a9c12d2d6a6..5f0a7d4b55d39 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -15,6 +15,10 @@ # which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give # it a count. # +# A dispatched run can also name the plugins to test instead of taking them from the directory, which is how to +# check a single plugin against a release candidate, or to reproduce a failure from an earlier run without +# waiting for the other plugins in its shard. +# # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. ## @@ -45,15 +49,22 @@ on: type: string default: 'nightly' plugin-count: - description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250. Ignored when plugin-slugs is given.' type: string default: '100' + plugin-slugs: + description: 'Optional. Test these WordPress.org plugin slugs instead of the most popular ones. Accepts a comma or space separated list, eg. "woocommerce, classic-editor".' + type: string + default: '' # Cancels all previous workflow runs for pull requests that have not completed. concurrency: # The concurrency group contains the workflow name and the branch name for pull requests # or the commit hash for any other events. - group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + # + # Dispatched runs are grouped by run instead, so that testing one plugin list does not cancel a run someone + # else started with a different one against the same version of WordPress. + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name == 'pull_request' && github.head_ref || github.sha }} cancel-in-progress: true # Disable permissions for all available scopes by default. @@ -64,25 +75,31 @@ jobs: # Builds the list of plugins to test and splits it into shards for the test matrix. # # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a - # smaller count still tests the plugins with the widest reach. + # smaller count still tests the plugins with the widest reach. A dispatched run can name the plugins to test + # instead, in which case the directory is not queried at all. # # Performs the following steps: - # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Queries the WordPress.org plugin directory API for the most popular plugins, or reads the plugin-slugs input. # - Splits the resulting slugs into shards and returns them as a job output. build-plugin-matrix: name: Build plugin matrix permissions: contents: read runs-on: ubuntu-24.04 - if: ${{ github.repository == 'WordPress/wordpress-develop' }} + # Dispatched runs are allowed anywhere so that this can be run from a fork, which is the only way for + # someone without write access to the repository to try it against a plugin of their choosing. + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} timeout-minutes: 5 outputs: shards: ${{ steps.plugin-shards.outputs.shards }} steps: - - name: Fetch the most popular plugins + - name: Build the list of plugins to test id: plugin-shards env: + # An explicit list of slugs to test in place of the most popular ones. Empty for everything but a + # dispatched run that filled the input in. + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test # this workflow rather than the ecosystem, so they use a small count. See the note on the # plugin-compatibility-tests job below. @@ -100,65 +117,94 @@ jobs: run: | set -euo pipefail - # Guard against a non-numeric value being passed to the API. - if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then - printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" - exit 1 - fi - - if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then - printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" - printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" - exit 1 - fi - RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" : > "${RAW_SLUGS}" - PAGE=1 - UNIQUE_COUNT=0 - - while : ; do - # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. - # The unneeded response fields are turned off to keep the payload small. - curl -sS --fail --retry 3 --retry-delay 5 \ - --get 'https://api.wordpress.org/plugins/info/1.2/' \ - --data-urlencode 'action=query_plugins' \ - --data-urlencode 'request[browse]=popular' \ - --data-urlencode "request[per_page]=${PAGE_SIZE}" \ - --data-urlencode "request[page]=${PAGE}" \ - --data-urlencode 'request[fields][short_description]=0' \ - --data-urlencode 'request[fields][sections]=0' \ - --data-urlencode 'request[fields][icons]=0' \ - --data-urlencode 'request[fields][banners]=0' \ - --data-urlencode 'request[fields][ratings]=0' \ - --data-urlencode 'request[fields][tags]=0' \ - --data-urlencode 'request[fields][compatibility]=0' \ - --data-urlencode 'request[fields][screenshots]=0' \ - -o "${RUNNER_TEMP}/plugins-page.json" - - PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" - jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" - - # Popularity ordering can shift between two requests, so the same slug can turn up on more than - # one page. Duplicates are dropped while the first occurrence keeps its position. + + if [ -n "${PLUGIN_SLUGS}" ]; then + # A named list replaces the popularity query entirely. Commas and whitespace both separate slugs, + # so a list can be pasted in however it was written down. + printf '%s' "${PLUGIN_SLUGS}" | tr ',[:space:]' '\n' > "${RAW_SLUGS}" awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" - UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" - printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + # A value outside the character set the directory uses is a typo worth stopping for now, rather + # than a plugin reported as impossible to download several minutes from now. + if INVALID_SLUGS="$( grep -Ev '^[a-z0-9-]+$' "${DEDUPED_SLUGS}" )"; then + printf 'The plugin-slugs input contains values that are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" + exit 1 + fi + + PLUGIN_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" - if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then - break + if [ "${PLUGIN_COUNT}" -lt 1 ]; then + printf 'The plugin-slugs input did not contain any plugin slugs.\n' + exit 1 fi - # A short page means the directory has nothing left to give. - if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then - printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" - break + if [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-slugs input names %s plugins, more than the %s this workflow will test in one run.\n' "${PLUGIN_COUNT}" "${MAX_PLUGIN_COUNT}" + exit 1 fi - PAGE=$(( PAGE + 1 )) - done + printf 'Testing the %s plugin(s) named in the plugin-slugs input.\n' "${PLUGIN_COUNT}" + else + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + fi SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" @@ -213,7 +259,7 @@ jobs: permissions: contents: read needs: [ build-plugin-matrix ] - if: ${{ github.repository == 'WordPress/wordpress-develop' }} + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} strategy: fail-fast: false matrix: From 2896efe087596ddad0b4a1d26f108775172dd185 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 31 Aug 2026 16:44:31 -0700 Subject: [PATCH 07/27] Build/Test Tools: Drop the removed Slack timeout webhook secret. `slack-notifications.yml` no longer accepts `SLACK_GHA_TIMEOUT_WEBHOOK`, and every other caller stopped passing it, so actionlint failed on this workflow once trunk was merged in. --- .github/workflows/plugin-compatibility.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 5f0a7d4b55d39..4a0655438904d 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -287,7 +287,6 @@ jobs: SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} - SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} failed-workflow: name: Failed workflow tasks From 69d0d379134ce40cf72b80ef12018dbb0394a48e Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 1 Sep 2026 08:56:01 -0700 Subject: [PATCH 08/27] Build/Test Tools: Make the plugin compatibility checks runnable from a terminal. Move the per-plugin checks out of the reusable workflow and into tools/plugin-compatibility/test-plugins.sh, so that reproducing a result from a run no longer means dispatching a workflow and waiting for it. The script provisions a database and a PHP with WP-CLI in throwaway containers by default, so nothing but Docker is needed locally. The workflow passes --no-docker and runs the same code against the runner's own PHP and database service, which keeps one copy of the logic that decides whether a plugin passed. --- .github/workflows/plugin-compatibility.yml | 6 + .../reusable-plugin-compatibility.yml | 447 +---------- tools/plugin-compatibility/test-plugins.sh | 709 ++++++++++++++++++ 3 files changed, 744 insertions(+), 418 deletions(-) create mode 100755 tools/plugin-compatibility/test-plugins.sh diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 4a0655438904d..a496df94681bd 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -19,6 +19,10 @@ # check a single plugin against a release candidate, or to reproduce a failure from an earlier run without # waiting for the other plugins in its shard. # +# The checks each shard runs live in `tools/plugin-compatibility/test-plugins.sh`. Running that script from a +# terminal runs exactly what a shard here runs, which is the quickest way to reproduce a result from a run +# without waiting on Actions. +# # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. ## @@ -32,6 +36,7 @@ on: paths: - '.github/workflows/plugin-compatibility.yml' - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' pull_request: # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. branches: @@ -40,6 +45,7 @@ on: paths: - '.github/workflows/plugin-compatibility.yml' - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' schedule: - cron: '0 2 * * 1' workflow_dispatch: diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index d32d2156b9b16..003146860563c 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -9,6 +9,10 @@ # that dependency is installed and active. Those dependencies are installed and activated alongside it, and the # site is checked with only the dependencies active first so that a dependency's own breakage is not reported # against the plugin under test. +# +# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that +# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top +# of that script. ## name: Plugin Compatibility Tests @@ -53,14 +57,9 @@ jobs: # Tests that each plugin in the shard can be activated against the given version of WordPress. # # Performs the following steps: - # - Sets up PHP. - # - Downloads the specified version of WordPress. - # - Creates a `wp-config.php` file with debugging and error logging enabled. - # - Installs WordPress. - # - Starts the PHP built-in web server so HTTP requests can be made against the site. - # - Installs, activates, exercises, and removes each plugin in the shard, one at a time, together with - # anything it declares in `Requires Plugins`. - # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + # - Checks out the repository, for the script that runs the checks. + # - Sets up PHP and WP-CLI. + # - Runs the checks against every plugin in the shard. plugin-compatibility-tests: name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} permissions: @@ -87,6 +86,14 @@ jobs: -e MYSQL_DATABASE="test_db" steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + - name: Set up PHP ${{ inputs.php-version }} uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: @@ -94,421 +101,25 @@ jobs: coverage: none tools: wp-cli - - name: Download WordPress ${{ inputs.wp-version }} - run: wp core download --version="${WP_VERSION}" - env: - WP_VERSION: ${{ inputs.wp-version }} - - - name: Create wp-config.php file - run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" - env: - DB_PORT: ${{ job.services.database.ports['3306'] }} - - # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. - # - # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a - # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. - # - # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by - # recovery mode, which would also deactivate the plugin mid-test. - # - # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The - # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned - # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is - # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. - - name: Enable debugging and error logging - run: | - wp config set WP_DEBUG true --raw - wp config set WP_DEBUG_LOG true --raw - wp config set WP_DEBUG_DISPLAY false --raw - wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw - wp config set DISABLE_WP_CRON true --raw - - - name: Install WordPress - run: | - wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ - --admin_password=password --admin_email=me@example.org --skip-email - env: - SITE_URL: http://127.0.0.1:8889 - - # The site needs to answer real requests so that fatals which only happen on a front end or admin page - # load are caught. The built-in server is enough for that and needs nothing installed. + # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the + # containers the script provisions when it is run from a terminal. # - # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of - # plugins make a loopback request to the site they are running on, and a single threaded server cannot - # answer one while it is still serving the request that made it. - - name: Start the PHP built-in web server - env: - PHP_CLI_SERVER_WORKERS: '4' - run: | - set -uo pipefail - - nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & - - # Wait for the server to start answering before any plugin is installed. - for _ in $( seq 1 30 ); do - if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then - printf 'The PHP built-in server is ready.\n' - exit 0 - fi - sleep 1 - done - - printf 'The PHP built-in server did not start.\n' - cat "${RUNNER_TEMP}/php-server.log" - exit 1 - + # WordPress is installed outside the workspace so that the checkout above is not part of the site under + # test. - name: Test each plugin in isolation env: PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} - PHP_VERSION: ${{ inputs.php-version }} - SITE_URL: http://127.0.0.1:8889 WP_VERSION: ${{ inputs.wp-version }} + DB_PORT: ${{ job.services.database.ports['3306'] }} run: | - # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining - # plugins in the shard from being tested. set -uo pipefail - RESULTS="${RUNNER_TEMP}/plugin-results.tsv" - RESPONSE_BODY="${RUNNER_TEMP}/response.html" - : > "${RESULTS}" - - # How many levels of `Requires Plugins` to follow. Chains in the directory are one level - # deep in practice, so the bound is only here to stop a circular declaration from looping - # forever. - MAX_DEPENDENCY_DEPTH=3 - - # Everything installed for the plugin currently under test, that plugin included. The - # directories are tracked alongside the slugs because a plugin does not always unpack into - # a directory named after its slug. - INSTALLED_SLUGS=() - INSTALLED_DIRS=() - - # record - # - # Appends one tab separated row to the results file. The reason is flattened so that it cannot - # break the markdown table that is generated from these rows later on. - record() { - SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" - printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" - } - - # check_url - # - # Requests a path on the test site and prints a reason to stdout when the response looks broken. - # Prints nothing when the request looks healthy. - check_url() { - CURL_EXIT_CODE=0 - # The exit code is captured separately rather than falling back to a literal inside the command - # substitution, which would append to whatever curl had already written and produce a nonsense - # status like "200000" when a request returned headers and then stalled. - HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? - - if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then - printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" - return - fi - - if [ "${HTTP_CODE}" -ge 500 ]; then - printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" - return - fi - - if [ "${CURL_EXIT_CODE}" -ne 0 ]; then - printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" - return - fi - - # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors - # back on for itself. - if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then - printf 'The response from %s contained a fatal error' "${1}" - fi - } - - # The plugins directory itself must never be removed, only directories inside it. - PLUGINS_ROOT="$( wp plugin path )" - - # install_plugin - # - # Installs a plugin from WordPress.org and remembers it so that it is removed again once the - # plugin under test has been checked. The slug is remembered before the download is attempted - # so that a partial download is still cleaned up. Returns non zero when the download failed. - install_plugin() { - INSTALLED_SLUGS+=( "${1}" ) - INSTALLED_DIRS+=( "" ) - - wp plugin install "${1}" --skip-plugins --skip-themes || return 1 - - INSTALLED_DIRS[-1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" - } - - # plugin_dependencies - # - # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' - # own header parser is used so that the same rules apply here as when core decides whether a - # plugin's dependencies are met. - plugin_dependencies() { - PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" - - [ -n "${PLUGIN_FILE}" ] || return 0 - - # The single quotes are what keep the shell out of the PHP below. The path is handed over as an - # environment variable rather than interpolated so that it never reaches PHP as source code. - # shellcheck disable=SC2016 - PLUGIN_FILE="${PLUGIN_FILE}" wp eval --skip-plugins --skip-themes ' - require_once ABSPATH . "wp-admin/includes/plugin.php"; - - $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); - - foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { - // Core only accepts a WordPress.org slug here, so anything else is ignored. - if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { - echo trim( $dependency ), "\n"; - } - } - ' 2>/dev/null - } - - # cleanup_plugins - # - # Returns the site to a clean slate by removing everything installed for the current plugin, - # its dependencies included. A plugin that fatals can take WP-CLI down with it, so every - # command here is allowed to fail and the plugin directory is removed directly as a fallback. - # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. - cleanup_plugins() { - for INDEX in "${!INSTALLED_SLUGS[@]}"; do - CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" - CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - - wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - - if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${CLEANUP_DIR}" - fi - - rm -rf "wp-content/plugins/${CLEANUP_SLUG}" - done - - INSTALLED_SLUGS=() - INSTALLED_DIRS=() - - # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. - wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true - } - - while IFS= read -r SLUG; do - [ -n "${SLUG}" ] || continue - - printf '::group::%s\n' "${SLUG}" - - STATUS="PASS" - REASON="-" - VERSION="unknown" - DEPENDENCIES=() - - # Start every plugin with an empty log so that anything found in it belongs to this plugin. - rm -f wp-content/debug.log - - # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, - # since it generally means a network flake or a plugin that is no longer in the directory. - if ! install_plugin "${SLUG}"; then - record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" - cleanup_plugins - printf '::endgroup::\n' - continue - fi - - VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" - - # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a - # plugin whose dependencies are missing, so without this every WooCommerce extension - a large - # slice of the most popular plugins - would go untested. - # - # Dependencies are resolved a level at a time so that a dependency declaring its own is - # installed too. Anything already installed is left alone, which covers a plugin named twice - # in the same tree as well as a declaration pointing back at the plugin under test. - mapfile -t PENDING < <( plugin_dependencies "${SLUG}" ) - DEPTH=0 - - while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do - NEXT=() - - for DEPENDENCY in "${PENDING[@]}"; do - if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then - continue - fi - - printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" - - if ! install_plugin "${DEPENDENCY}"; then - STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" - break 2 - fi - - DEPENDENCIES+=( "${DEPENDENCY}" ) - mapfile -t -O "${#NEXT[@]}" NEXT < <( plugin_dependencies "${DEPENDENCY}" ) - done - - PENDING=( "${NEXT[@]}" ) - DEPTH=$(( DEPTH + 1 )) - done - - # Step 3: activate the dependencies, deepest first. That is the reverse of the order they - # were discovered in, and it matters because core will not activate a plugin ahead of its own - # requirements either. - if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do - DEPENDENCY="${DEPENDENCIES[${INDEX}]}" - DEPENDENCY_EXIT_CODE=0 - DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? - printf '%s\n' "${DEPENDENCY_OUTPUT}" - - if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then - STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} could not be activated" - break - fi - done - fi - - # Step 4: with dependencies active the plugin under test is no longer alone on the site, so - # the baseline is checked before it is activated. A site that is already broken says something - # about the dependency rather than about the plugin being tested, and blaming the plugin for - # it would be the masking this workflow is built to avoid. - if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - for URL_PATH in "/" "/wp-login.php"; do - BASELINE_REASON="$( check_url "${URL_PATH}" )" - - if [ -n "${BASELINE_REASON}" ]; then - STATUS="SKIPPED" - REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" - break - fi - done - - # Anything the dependencies logged on their way up is not the responsibility of the plugin - # under test, so the log starts empty again here. - rm -f wp-content/debug.log - fi - - # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. - if [ "${STATUS}" = "PASS" ]; then - ACTIVATE_EXIT_CODE=0 - ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? - printf '%s\n' "${ACTIVATE_OUTPUT}" - - if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then - case "${ACTIVATE_OUTPUT}" in - # Core refuses to activate a plugin whose declared requirements are not met. Dependencies - # are installed above, so what is left here is a plugin asking for a version of PHP or - # WordPress this job is not running, or for a dependency that is not on WordPress.org. - # That is core working as designed rather than a fatal, so it is recorded as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) - STATUS="SKIPPED" - REASON="Core declined to activate the plugin because its declared requirements are not met" - ;; - * ) - STATUS="FAIL" - REASON="The plugin could not be activated" - ;; - esac - fi - fi - - # Step 6: boot all of core plus the active plugin in a CLI context. - if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true - - if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then - printf '%s\n' "${EVAL_OUTPUT}" - - case "${EVAL_OUTPUT}" in - *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) - STATUS="FAIL" - REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" - ;; - # Some plugins redirect or exit while loading, which stops WP-CLI without anything being - # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and - # debug log checks below, which see the same code in a real request. - * ) - printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' - ;; - esac - fi - fi - - # Step 7: request the front page and the login screen through the PHP built-in server. - if [ "${STATUS}" = "PASS" ]; then - for URL_PATH in "/" "/wp-login.php"; do - HTTP_REASON="$( check_url "${URL_PATH}" )" - - if [ -n "${HTTP_REASON}" ]; then - STATUS="FAIL" - REASON="${HTTP_REASON}" - break - fi - done - fi - - # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown - # hook, so the debug log is checked separately. - if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then - grep 'PHP Fatal' wp-content/debug.log - STATUS="FAIL" - REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" - fi - - # Step 9: record the outcome and put the site back the way it was found. - DEPENDENCY_LIST="-" - - if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" - DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" - fi - - record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" - cleanup_plugins - - printf '%s: %s\n' "${SLUG}" "${STATUS}" - printf '::endgroup::\n' - done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) - - PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" - FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" - SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" - - { - printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" - printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" - printf '| Plugin | Version | Result | Also active | Details |\n' - printf '| --- | --- | --- | --- | --- |\n' - - while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_DEPENDENCIES ROW_REASON; do - case "${ROW_STATUS}" in - PASS ) ICON=':white_check_mark:' ;; - FAIL ) ICON=':x:' ;; - * ) ICON=':warning:' ;; - esac - - printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s | %s |\n' \ - "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_DEPENDENCIES}" "${ROW_REASON}" - done < "${RESULTS}" - - printf '\n' - } >> "${GITHUB_STEP_SUMMARY}" - - # Plugins that could not be downloaded are reported but do not fail the run. - if [ "${FAIL_COUNT}" -gt 0 ]; then - printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" - awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" - exit 1 - fi - - printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" - - - name: Show the web server log - if: ${{ failure() }} - run: cat "${RUNNER_TEMP}/php-server.log" + ./tools/plugin-compatibility/test-plugins.sh \ + --no-docker \ + --wp-version="${WP_VERSION}" \ + --dir="${RUNNER_TEMP}/wordpress" \ + --db-host="127.0.0.1:${DB_PORT}" \ + --db-name=test_db \ + --db-user=root \ + --db-pass=root \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh new file mode 100755 index 0000000000000..3ecafaaa1f561 --- /dev/null +++ b/tools/plugin-compatibility/test-plugins.sh @@ -0,0 +1,709 @@ +#!/usr/bin/env bash +## +# Checks that plugins from the WordPress.org directory can be activated against a version of WordPress +# without fataling. +# +# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then removed +# again before the next one is installed, so that one broken plugin cannot mask (or break) the next. A plugin +# that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core +# refuses to activate it otherwise. +# +# This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a +# failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. +# +# By default everything is provisioned in throwaway containers - a database, and a PHP with WP-CLI to run the +# checks in - so nothing but Docker is needed and nothing is left behind: +# +# tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor +# +# Pass `--no-docker` to run the checks in the current environment instead, against a database that is already +# running. That is the path the workflow takes, where the runner already has PHP, WP-CLI and a database +# service. +# +# Usage: test-plugins.sh [options] [slug...] +# +# Run `test-plugins.sh --help` for the options. +## + +# `set -e` is deliberately not used: a plugin that fatals must not stop the remaining plugins from being +# tested. Failures are checked where they matter instead. +set -uo pipefail + +# Written for Bash 3.2 so that it runs on a stock macOS as well as on a runner. + +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null 2>&1 && pwd )/$( basename "${BASH_SOURCE[0]}" )" + +# Defaults. Every one of these can be overridden with the matching option. +WP_VERSION="nightly" +PLUGIN_LIST="" +USE_DOCKER="yes" +# The official WP-CLI image carries PHP, WP-CLI, `mysqli` and curl, which is everything the checks need. +PHP_IMAGE="wordpress:cli-php8.3" +# Matches the database the workflow runs against by default. +DB_IMAGE="mysql:8.4" +WP_DIR="" +DB_HOST="127.0.0.1" +DB_NAME="test_db" +DB_USER="root" +DB_PASS="password" +SERVER_PORT="8889" +KEEP_INSTALL="no" + +# How many levels of `Requires Plugins` to follow. Chains in the directory are one level deep in practice, so +# the bound is only here to stop a circular declaration from looping forever. +MAX_DEPENDENCY_DEPTH=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-plugins.sh [options] [slug...] + +Checks that WordPress.org plugins can be activated against a version of WordPress without fataling. + +Options: + --plugins= Plugin slugs to test, separated by commas or spaces. Slugs can also be passed as + positional arguments. + --wp-version= Version of WordPress to test against. Accepts "latest", "nightly", or a version + number such as a beta or RC. Default: nightly. + --php-image= Container image the checks run in. Default: wordpress:cli-php8.3. + --db-image= Container image the database runs in. Default: mysql:8.4. + --no-docker Run the checks in the current environment rather than in a container. Requires PHP, + WP-CLI, curl and a database that is already running. + --dir= Where to install WordPress. Only used with --no-docker. Default: a temporary + directory, which is removed again on exit. + --db-host= Database host, with an optional port. Only used with --no-docker. Default: 127.0.0.1. + --db-name= Database name. Only used with --no-docker. Default: test_db. + --db-user= Database user. Only used with --no-docker. Default: root. + --db-pass= Database password. Only used with --no-docker. Default: password. + --port= Port the PHP built-in server listens on. Default: 8889. + --keep Leave the WordPress install in place on exit rather than removing it. Only used with + --no-docker. + -h, --help Print this message. + +Examples: + # Check two plugins against nightly, provisioning everything in containers. + test-plugins.sh --wp-version=nightly woocommerce classic-editor + + # Reproduce a failure from a workflow run against the release it was seen on. + test-plugins.sh --wp-version=7.1-RC1 eps-301-redirects + + # Run the checks here, against a database that is already running. + test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + +Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core +declined to activate because its declared requirements are not met, is reported as skipped and does not +fail the run. +USAGE_EOF +} + +# error +# +# Prints a message to stderr and stops. +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --php-image=* ) PHP_IMAGE="${1#*=}" ;; + --db-image=* ) DB_IMAGE="${1#*=}" ;; + --no-docker ) USE_DOCKER="no" ;; + --dir=* ) WP_DIR="${1#*=}" ;; + --db-host=* ) DB_HOST="${1#*=}" ;; + --db-name=* ) DB_NAME="${1#*=}" ;; + --db-user=* ) DB_USER="${1#*=}" ;; + --db-pass=* ) DB_PASS="${1#*=}" ;; + --port=* ) SERVER_PORT="${1#*=}" ;; + --keep ) KEEP_INSTALL="yes" ;; + -h | --help ) usage; exit 0 ;; + -* ) usage >&2; error "Unknown option ${1}." ;; + * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; + esac + shift +done + +# Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. +PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" + +if [ -z "${PLUGIN_SLUGS}" ]; then + usage >&2 + error "No plugins were named." +fi + +# A value outside the character set the directory uses is a typo worth stopping for, rather than a plugin +# reported as impossible to download several minutes from now. +INVALID_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | grep -Ev '^[a-z0-9-]+$' )" + +if [ -n "${INVALID_SLUGS}" ]; then + printf 'These values are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" >&2 + exit 1 +fi + +## +# Provisions a database and a PHP, and runs this same script inside them. +# +# The checks themselves are identical either way: the container is handed this file and runs it with +# `--no-docker`, so there is only one copy of the logic that decides whether a plugin passed. +## +run_in_docker() { + command -v docker > /dev/null 2>&1 || error "Docker is required, or pass --no-docker to run the checks here." + + # Everything is named after the process so that two runs at once do not collide. + DOCKER_NETWORK="plugin-compatibility-$$" + DB_CONTAINER="plugin-compatibility-db-$$" + + # Both are removed however this exits, an interrupt included. + trap 'docker rm --force "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM + + printf 'Starting %s.\n' "${DB_IMAGE}" + + docker network create "${DOCKER_NETWORK}" > /dev/null || error "The Docker network could not be created." + + docker run --detach --name "${DB_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --env MYSQL_ROOT_PASSWORD="${DB_PASS}" \ + --env MYSQL_DATABASE="${DB_NAME}" \ + "${DB_IMAGE}" > /dev/null || error "The database container could not be started." + + # A first run has to initialise the data directory, which takes appreciably longer than a later one. + DB_READY="no" + + for _ in $( seq 1 90 ); do + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + DB_READY="yes" + break + fi + sleep 2 + done + + if [ "${DB_READY}" != "yes" ]; then + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 20 + error "The database did not become available." + fi + + printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot + # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a + # writable directory because WP-CLI caches its downloads there. + docker run --rm --network "${DOCKER_NETWORK}" \ + --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + --env HOME=/tmp \ + --entrypoint bash \ + "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ + --no-docker \ + --dir=/tmp/wordpress \ + --wp-version="${WP_VERSION}" \ + --db-host="${DB_CONTAINER}" \ + --db-name="${DB_NAME}" \ + --db-user=root \ + --db-pass="${DB_PASS}" \ + --port="${SERVER_PORT}" \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | tr '\n' ' ' )" +} + +## +# Installs WordPress, starts a web server, and checks each plugin against it. +## +run_checks() { + for REQUIRED_COMMAND in php curl; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required to run the checks without Docker." + done + + # `type -P` searches the path rather than resolving to the `wp` wrapper function defined below. + WP_CLI_BIN="$( type -P wp )" + + [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + + WORK_DIR="$( mktemp -d )" + RESULTS="${WORK_DIR}/results.tsv" + RESPONSE_BODY="${WORK_DIR}/response.html" + SERVER_LOG="${WORK_DIR}/php-server.log" + SERVER_PID="" + : > "${RESULTS}" + + if [ -z "${WP_DIR}" ]; then + WP_DIR="${WORK_DIR}/wordpress" + fi + + mkdir -p "${WP_DIR}" || error "The WordPress directory could not be created." + + # The server is stopped and the temporary files are removed however this exits. The WordPress install is + # only removed when this script created it, so that `--dir` never deletes a directory it was handed. + cleanup_environment() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" > /dev/null 2>&1 + fi + + if [ "${KEEP_INSTALL}" = "yes" ]; then + printf '\nThe WordPress install was left in %s.\n' "${WP_DIR}" + else + rm -rf "${WORK_DIR}" + fi + } + + trap cleanup_environment EXIT INT TERM + + cd "${WP_DIR}" || error "The WordPress directory could not be entered." + + SITE_URL="http://127.0.0.1:${SERVER_PORT}" + + printf 'Downloading WordPress %s.\n' "${WP_VERSION}" + wp core download --version="${WP_VERSION}" || error "WordPress ${WP_VERSION} could not be downloaded." + + wp config create --dbname="${DB_NAME}" --dbuser="${DB_USER}" --dbpass="${DB_PASS}" --dbhost="${DB_HOST}" \ + || error "wp-config.php could not be written." + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed + # by recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that + # spawned it, and the two deadlock until curl gives up. + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email \ + || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + + SERVER_READY="no" + + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "${SITE_URL}/wp-login.php"; then + SERVER_READY="yes" + break + fi + sleep 1 + done + + if [ "${SERVER_READY}" != "yes" ]; then + cat "${SERVER_LOG}" + error "The PHP built-in server did not start on port ${SERVER_PORT}." + fi + + printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" + + test_plugins + report_results +} + +# wp +# +# Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock +# php.ini allows, and WP-CLI reads its own `WP_CLI_PHP_ARGS` only when it is installed as a wrapper script +# rather than as the phar most environments carry. +wp() { + php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" +} + +# record +# +# Appends one tab separated row to the results file. The reason is flattened so that it cannot break the +# markdown table that is generated from these rows later on. +record() { + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" +} + +# check_url +# +# Requests a path on the test site and prints a reason to stdout when the response looks broken. Prints +# nothing when the request looks healthy. +check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense status + # like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for + # itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi +} + +# install_plugin +# +# Installs a plugin from WordPress.org and remembers it so that it is removed again once the plugin under +# test has been checked. The slug is remembered before the download is attempted so that a partial download +# is still cleaned up. Returns non zero when the download failed. +install_plugin() { + INSTALLED_SLUGS[${#INSTALLED_SLUGS[@]}]="${1}" + INSTALLED_DIRS[${#INSTALLED_DIRS[@]}]="" + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[${#INSTALLED_DIRS[@]} - 1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" +} + +# plugin_dependencies +# +# Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header +# parser is used so that the same rules apply here as when core decides whether a plugin's dependencies are +# met. +plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + export PLUGIN_FILE + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null +} + +# cleanup_plugins +# +# Returns the site to a clean slate by removing everything installed for the current plugin, its +# dependencies included. A plugin that fatals can take WP-CLI down with it, so every command here is allowed +# to fail and the plugin directory is removed directly as a fallback. `--skip-plugins` keeps WP-CLI from +# loading the broken plugin while cleaning up after it. +cleanup_plugins() { + if [ "${#INSTALLED_SLUGS[@]}" -gt 0 ]; then + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" + + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "${WP_DIR}/wp-content/plugins/${CLEANUP_SLUG}" + done + fi + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 +} + +# group_start / group_end +# +# Collapses a plugin's output in the workflow log. The markers mean nothing to a terminal, so they are only +# printed when this is running in Actions. +group_start() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::group::%s\n' "${1}" + else + printf '\n----- %s -----\n' "${1}" + fi +} + +group_end() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::endgroup::\n' + fi +} + +## +# Tests each plugin in turn, writing a row per plugin to the results file. +## +test_plugins() { + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # Everything installed for the plugin currently under test, that plugin included. The directories are + # tracked alongside the slugs because a plugin does not always unpack into a directory named after its + # slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + group_start "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + DEPENDENCIES=() + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f "${WP_DIR}/wp-content/debug.log" + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, since + # it generally means a network flake or a plugin that is no longer in the directory. + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins + group_end + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a plugin + # whose dependencies are missing, so without this every WooCommerce extension - a large slice of the + # most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is installed + # too. Anything already installed is left alone, which covers a plugin named twice in the same tree + # as well as a declaration pointing back at the plugin under test. + PENDING=() + + while IFS= read -r DEPENDENCY; do + [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" + done < <( plugin_dependencies "${SLUG}" ) + + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" + + while IFS= read -r NESTED_DEPENDENCY; do + [ -n "${NESTED_DEPENDENCY}" ] && NEXT[${#NEXT[@]}]="${NESTED_DEPENDENCY}" + done < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( ${NEXT[@]+"${NEXT[@]}"} ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they were + # discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so the + # baseline is checked before it is activated. A site that is already broken says something about the + # dependency rather than about the plugin being tested, and blaming the plugin for it would be the + # masking these checks are built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin under + # test, so the log starts empty again here. + rm -f "${WP_DIR}/wp-content/debug.log" + fi + + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. + # Dependencies are installed above, so what is left here is a plugin asking for a + # version of PHP or WordPress this run is not using, or for a dependency that is not on + # WordPress.org. That is core working as designed rather than a fatal, so it is recorded + # as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 7: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then + grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + fi + + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + group_end + done <> "${GITHUB_STEP_SUMMARY}" + fi + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf '\nThe following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" + return 1 + fi + + printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" +} + +if [ "${USE_DOCKER}" = "yes" ]; then + run_in_docker + exit $? +fi + +run_checks From a8abc6e757b98fad550947770d03a5cc32254e2e Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 1 Sep 2026 09:37:16 -0700 Subject: [PATCH 09/27] Build/Test Tools: Let a real request decide whether a plugin passed. WP-CLI requires wp-settings.php from inside a method, so a plugin that assigns a variable at file scope and reads it back with global later finds nothing there. eps-301-redirects does exactly that and fatals under WP-CLI on a fresh install, while the front page, the login screen and the debug log are all clean. Failing it reported breakage that no visitor would ever see. Move the WP-CLI boot check after the HTTP and debug log checks and stop letting it fail a plugin on its own. A plugin that is healthy over HTTP now passes with the WP-CLI fatal recorded as a note. Anything that genuinely fatals on load still fails, because it fails the HTTP checks. The reorder also matters because the WP-CLI fatal is written to the debug log, which the log check would otherwise pick up as a failure. --- tools/plugin-compatibility/test-plugins.sh | 60 +++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 3ecafaaa1f561..a225ce799a752 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -8,6 +8,9 @@ # that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core # refuses to activate it otherwise. # +# What a real request does is what decides whether a plugin passed. The WP-CLI check runs last and only ever +# adds a note, because WP-CLI loads WordPress in a way a visitor never does. +# # This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a # failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. # @@ -590,29 +593,8 @@ test_plugins() { fi fi - # Step 6: boot all of core plus the active plugin in a CLI context. - if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" - - if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then - printf '%s\n' "${EVAL_OUTPUT}" - - case "${EVAL_OUTPUT}" in - *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) - STATUS="FAIL" - REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" - ;; - # Some plugins redirect or exit while loading, which stops WP-CLI without anything being - # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and - # debug log checks below, which see the same code in a real request. - * ) - printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' - ;; - esac - fi - fi - - # Step 7: request the front page and the login screen through the PHP built-in server. + # Step 6: request the front page and the login screen through the PHP built-in server. A real request + # is what decides whether a plugin passed, because it is what a visitor gets. if [ "${STATUS}" = "PASS" ]; then for URL_PATH in "/" "/wp-login.php"; do HTTP_REASON="$( check_url "${URL_PATH}" )" @@ -625,7 +607,7 @@ test_plugins() { done fi - # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown + # Step 7: a fatal can be logged without changing the HTTP status, for example during a shutdown # hook, so the debug log is checked separately. if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" @@ -633,6 +615,36 @@ test_plugins() { REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" fi + # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a + # plugin on its own. + # + # WP-CLI requires `wp-settings.php` from inside a method, so a plugin that assigns a variable at file + # scope and reads it back with `global` later finds nothing there. Plenty of plugins do exactly that, + # and the resulting fatal happens only under WP-CLI - a visitor never sees it. Failing a plugin for + # it would report breakage that does not exist on a real site, so it is recorded as a note against a + # plugin that is otherwise healthy. Anything that genuinely fatals on load fails the HTTP checks + # above. + # + # The fatal is written to the debug log too, which is why this comes after the log has been checked. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + # Step 9: record the outcome and put the site back the way it was found. DEPENDENCY_LIST="-" From 3e36d532ba05cca88622370aa5ff46f3f7eadcd9 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 29 Sep 2026 15:45:09 -0700 Subject: [PATCH 10/27] Build/Test Tools: Stop every PHP server worker when the checks finish. The built-in server forks the workers PHP_CLI_SERVER_WORKERS asks for, and killing the parent alone left them serving on the port. A --no-docker run straight after could then hit the old site or fail its readiness check. The server now starts in its own process group, which is signalled as a whole and waited on until every worker has exited. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01211BLKtDajir1Xg7etxT7R --- tools/plugin-compatibility/test-plugins.sh | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index a225ce799a752..b8751666b1346 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -236,7 +236,7 @@ run_checks() { # only removed when this script created it, so that `--dir` never deletes a directory it was handed. cleanup_environment() { if [ -n "${SERVER_PID}" ]; then - kill "${SERVER_PID}" > /dev/null 2>&1 + stop_server fi if [ "${KEEP_INSTALL}" = "yes" ]; then @@ -246,6 +246,23 @@ run_checks() { fi } + # The built-in server forks its workers, and they keep answering on the port after the process that + # started them is gone. The server runs in its own process group (see below) so that the whole group can + # be stopped at once, and this waits for the last of them to exit so that a run which follows straight + # after finds the port free. + stop_server() { + kill -TERM -- "-${SERVER_PID}" > /dev/null 2>&1 + + for _ in $( seq 1 50 ); do + kill -0 -- "-${SERVER_PID}" > /dev/null 2>&1 || break + sleep 0.1 + done + + kill -KILL -- "-${SERVER_PID}" > /dev/null 2>&1 + wait "${SERVER_PID}" > /dev/null 2>&1 + SERVER_PID="" + } + trap cleanup_environment EXIT INT TERM cd "${WP_DIR}" || error "The WordPress directory could not be entered." @@ -285,8 +302,13 @@ run_checks() { # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of # plugins make a loopback request to the site they are running on, and a single threaded server cannot # answer one while it is still serving the request that made it. + # + # Job control is switched on just long enough to start the server, because that is what gives a + # background job a process group of its own, whose ID is the PID of the server. + set -m PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & SERVER_PID=$! + set +m SERVER_READY="no" From 67de68b241f914f49f95299149b2cfad76e385f9 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 29 Sep 2026 15:46:28 -0700 Subject: [PATCH 11/27] Build/Test Tools: Activate plugin dependencies in passes. Reversing the discovery order is not dependency order. A plugin requiring `base, extension`, where extension also requires base, had extension activated first; core refused it and the plugin under test was recorded as SKIPPED without ever running. Each pass now activates whatever core accepts and retries the rest, and only a pass that activates nothing marks the plugin SKIPPED. That still ends a broken or circular dependency set, bounded by the number of dependencies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01211BLKtDajir1Xg7etxT7R --- tools/plugin-compatibility/test-plugins.sh | 29 ++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index b8751666b1346..3238774d54585 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -552,21 +552,30 @@ test_plugins() { DEPTH=$(( DEPTH + 1 )) done - # Step 3: activate the dependencies, deepest first. That is the reverse of the order they were - # discovered in, and it matters because core will not activate a plugin ahead of its own - # requirements either. + # Step 3: activate the dependencies. Core will not activate a plugin ahead of its own requirements, + # and the order they were discovered in says nothing about the order they depend on each other in - + # a plugin can require `base, extension` where `extension` requires `base` as well. So they are + # activated in passes: each pass activates whatever it can, and whatever core declined is tried + # again on the next. A pass that activates nothing means the rest can never be activated, whether + # because one of them is broken or because they require each other. if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do - DEPENDENCY="${DEPENDENCIES[${INDEX}]}" - DEPENDENCY_EXIT_CODE=0 - DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? - printf '%s\n' "${DEPENDENCY_OUTPUT}" + INACTIVE=( "${DEPENDENCIES[@]}" ) - if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + while [ "${#INACTIVE[@]}" -gt 0 ]; do + STILL_INACTIVE=() + + for DEPENDENCY in "${INACTIVE[@]}"; do + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || STILL_INACTIVE[${#STILL_INACTIVE[@]}]="${DEPENDENCY}" + printf '%s\n' "${DEPENDENCY_OUTPUT}" + done + + if [ "${#STILL_INACTIVE[@]}" -eq "${#INACTIVE[@]}" ]; then STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} could not be activated" + REASON="The required plugin ${STILL_INACTIVE[0]} could not be activated" break fi + + INACTIVE=( ${STILL_INACTIVE[@]+"${STILL_INACTIVE[@]}"} ) done fi From cabe3074c213d71f8bcb4bd6dc7ca7bc7416c0f3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 29 Sep 2026 15:46:28 -0700 Subject: [PATCH 12/27] Build/Test Tools: Check the dependency baseline log for fatals. A dependency that fatals after output has started still answers with HTTP 200, so it passed the baseline requests. The log was then cleared unread, and when the same fatal was logged again with the plugin under test active, that plugin was recorded as FAIL. The baseline log is now checked before it is cleared, and a fatal there marks the plugin SKIPPED against its dependencies, the same as a failed baseline request. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01211BLKtDajir1Xg7etxT7R --- tools/plugin-compatibility/test-plugins.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 3238774d54585..195c7f655cfb9 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -594,8 +594,17 @@ test_plugins() { fi done - # Anything the dependencies logged on their way up is not the responsibility of the plugin under - # test, so the log starts empty again here. + # A dependency can fatal after the page has started to render, which still answers with a 200. + # The fatal only shows up in the log, and it would show up again once the plugin under test is + # active and be blamed on that plugin, so the log is checked before it is cleared. + if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then + grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + fi + + # Anything else the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. rm -f "${WP_DIR}/wp-content/debug.log" fi From 6ef62efb0bc3eeb0eec94d50bd53c09d5cee260d Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 09:02:19 -0800 Subject: [PATCH 13/27] Build/Test Tools: Clean up after local plugin compatibility runs. A relative --dir broke once the script changed into it, because the server document root, the debug log and the cleanup all resolved the path again from inside it. The directory is now made absolute as soon as it is entered. The database container was removed without its anonymous data volume, so every local run left one behind. It is now removed with --volumes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 195c7f655cfb9..57ce1a0231829 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -157,8 +157,9 @@ run_in_docker() { DOCKER_NETWORK="plugin-compatibility-$$" DB_CONTAINER="plugin-compatibility-db-$$" - # Both are removed however this exits, an interrupt included. - trap 'docker rm --force "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM + # Both are removed however this exits, an interrupt included. `--volumes` takes the anonymous volume the + # database image keeps its data in along with the container, which would otherwise pile up run after run. + trap 'docker rm --force --volumes "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM printf 'Starting %s.\n' "${DB_IMAGE}" @@ -267,6 +268,9 @@ run_checks() { cd "${WP_DIR}" || error "The WordPress directory could not be entered." + # A relative `--dir` would be resolved against the wrong directory from here on. + WP_DIR="$( pwd )" + SITE_URL="http://127.0.0.1:${SERVER_PORT}" printf 'Downloading WordPress %s.\n' "${WP_VERSION}" From 59a70d726c3ec47a20131786a56962439db8db9f Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 09:02:37 -0800 Subject: [PATCH 14/27] Build/Test Tools: Make sure the plugin checks reach their own server. When the port was already taken, the new PHP server exited with "Address in use" but the readiness request was answered by whatever owned the port, so every check that followed ran against an unrelated site and could pass. Readiness now requires the new server to still be running and to serve a file with a random token written into this install only. A port that is taken stops the run with an error naming the cause. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 57ce1a0231829..917124b4fd5fc 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -314,19 +314,30 @@ run_checks() { SERVER_PID=$! set +m + # Any server already listening on the port would answer a request for a WordPress page just as well, and + # every check after this would then be made against that site instead. So readiness means this server is + # still running and is serving a file that only exists in this install. + READY_FILE="plugin-compatibility-ready-$$-${RANDOM}.txt" + READY_TOKEN="$$-${RANDOM}-$( date +%s )" + printf '%s' "${READY_TOKEN}" > "${WP_DIR}/${READY_FILE}" + SERVER_READY="no" for _ in $( seq 1 30 ); do - if curl -sSf -o /dev/null "${SITE_URL}/wp-login.php"; then + kill -0 "${SERVER_PID}" > /dev/null 2>&1 || break + + if [ "$( curl -sf "${SITE_URL}/${READY_FILE}" 2>/dev/null )" = "${READY_TOKEN}" ]; then SERVER_READY="yes" break fi sleep 1 done + rm -f "${WP_DIR:?}/${READY_FILE}" + if [ "${SERVER_READY}" != "yes" ]; then cat "${SERVER_LOG}" - error "The PHP built-in server did not start on port ${SERVER_PORT}." + error "The PHP built-in server did not start on port ${SERVER_PORT}, or something else is already listening on it." fi printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" From aa19c04e92a5ebe367221737910f38c0df073d44 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 09:02:54 -0800 Subject: [PATCH 15/27] Build/Test Tools: Treat logged parse errors as fatals in plugin checks. A file with a syntax error that is included after output has started ends the request with a 200 and logs "PHP Parse error", which neither the response checks nor the log checks looked for, so the plugin passed. Both log checks now match parse errors as well as fatals through one helper, keeping their separate attribution: a dependency's error still skips the plugin, and the plugin's own still fails it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 29 ++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 917124b4fd5fc..95349090a4d7c 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -392,11 +392,20 @@ check_url() { # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for # itself. - if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + if grep -qiE '(Fatal|Parse) error' "${RESPONSE_BODY}"; then printf 'The response from %s contained a fatal error' "${1}" fi } +# logged_fatals +# +# Prints the fatal and parse errors in the debug log, one per line, and returns non zero when there are none. +# A parse error in a file included after output has started ends the request just as surely as a fatal does, +# and still answers with a 200. +logged_fatals() { + [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -E 'PHP (Fatal|Parse) error' "${WP_DIR}/wp-content/debug.log" +} + # install_plugin # # Installs a plugin from WordPress.org and remembers it so that it is removed again once the plugin under @@ -612,10 +621,10 @@ test_plugins() { # A dependency can fatal after the page has started to render, which still answers with a 200. # The fatal only shows up in the log, and it would show up again once the plugin under test is # active and be blamed on that plugin, so the log is checked before it is cleared. - if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then - grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" STATUS="SKIPPED" - REASON="The required plugins are not healthy on their own: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + REASON="The required plugins are not healthy on their own: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" fi # Anything else the dependencies logged on their way up is not the responsibility of the plugin @@ -662,12 +671,12 @@ test_plugins() { done fi - # Step 7: a fatal can be logged without changing the HTTP status, for example during a shutdown - # hook, so the debug log is checked separately. - if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then - grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + # Step 7: a fatal or parse error can be logged without changing the HTTP status, for example during + # a shutdown hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" STATUS="FAIL" - REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + REASON="A fatal error was logged: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" fi # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a @@ -688,7 +697,7 @@ test_plugins() { printf '%s\n' "${EVAL_OUTPUT}" case "${EVAL_OUTPUT}" in - *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + *"Fatal error"* | *"Parse error"* | *"Uncaught"* ) REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" ;; # Some plugins redirect or exit while loading, which stops WP-CLI without anything being From 2f1bf7cc739066ae0ba8a62189a113150f12b0fe Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 09:03:22 -0800 Subject: [PATCH 16/27] Build/Test Tools: Confirm plugins are active before checking the site. A plugin that calls exit from its activation hook ends WP-CLI with a zero status before core records it as active. The checks then ran against a site without the plugin and passed it. Activation is now confirmed with wp plugin is-active for the plugin under test and for each dependency; a plugin that never became active is skipped, and a dependency that never did is retried in the next activation pass like one that was refused. A dependency that was already installed, such as the Akismet bundled with core, was treated as resolved and never activated or followed, so a plugin requiring it was skipped as having unmet requirements. Dependencies are now tracked by whether they have been resolved rather than whether they are installed: an installed one is not downloaded again but is still activated, baseline checked, and traversed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 46 ++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 95349090a4d7c..38b9342ca8c60 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -420,6 +420,13 @@ install_plugin() { INSTALLED_DIRS[${#INSTALLED_DIRS[@]} - 1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" } +# is_active +# +# Returns zero when core has recorded the plugin as active. +is_active() { + wp plugin is-active "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 +} + # plugin_dependencies # # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header @@ -539,9 +546,11 @@ test_plugins() { # most popular plugins - would go untested. # # Dependencies are resolved a level at a time so that a dependency declaring its own is installed - # too. Anything already installed is left alone, which covers a plugin named twice in the same tree - # as well as a declaration pointing back at the plugin under test. + # too. Each one is resolved only once, which covers a plugin named twice in the same tree as well as a + # declaration pointing back at the plugin under test. A dependency that is already installed, such as + # the Akismet that ships with core, is not downloaded again but is still activated and followed. PENDING=() + RESOLVED=$'\n'"${SLUG}"$'\n' while IFS= read -r DEPENDENCY; do [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" @@ -553,16 +562,20 @@ test_plugins() { NEXT=() for DEPENDENCY in "${PENDING[@]}"; do - if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then - continue - fi + case "${RESOLVED}" in + *$'\n'"${DEPENDENCY}"$'\n'* ) continue ;; + esac - printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + RESOLVED="${RESOLVED}${DEPENDENCY}"$'\n' - if ! install_plugin "${DEPENDENCY}"; then - STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" - break 2 + if ! wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi fi DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" @@ -582,6 +595,9 @@ test_plugins() { # activated in passes: each pass activates whatever it can, and whatever core declined is tried # again on the next. A pass that activates nothing means the rest can never be activated, whether # because one of them is broken or because they require each other. + # + # A successful exit is not taken as proof of activation. A plugin that calls `exit` while activating + # ends WP-CLI with a zero status before core has recorded it as active. if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then INACTIVE=( "${DEPENDENCIES[@]}" ) @@ -589,8 +605,12 @@ test_plugins() { STILL_INACTIVE=() for DEPENDENCY in "${INACTIVE[@]}"; do - DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || STILL_INACTIVE[${#STILL_INACTIVE[@]}]="${DEPENDENCY}" + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if ! is_active "${DEPENDENCY}"; then + STILL_INACTIVE[${#STILL_INACTIVE[@]}]="${DEPENDENCY}" + fi done if [ "${#STILL_INACTIVE[@]}" -eq "${#INACTIVE[@]}" ]; then @@ -654,6 +674,10 @@ test_plugins() { REASON="The plugin could not be activated" ;; esac + elif ! is_active "${SLUG}"; then + # Checking the site now would be checking it without the plugin, and passing it for that. + STATUS="SKIPPED" + REASON="WP-CLI reported success, but the plugin was not active afterwards, so its activation stopped early" fi fi From f2c556bfdf779a1e06b1f03ec9664396ab651240 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 09:08:34 -0800 Subject: [PATCH 17/27] Build/Test Tools: Reset the whole site between plugin checks. Removing plugin directories and emptying active_plugins left behind everything else a plugin can write: drop-ins, must-use plugins, wp-config.php constants, files in the install root, tables and options. A must-use loader left requiring a deleted plugin broke every plugin tested after it, which a run against fixtures reproduced as a download SKIPPED and an activation FAIL for two healthy plugins. The fresh install is now recorded once, as an archive of wp-config.php and wp-content, a list of root entries and a copy of each table in the same database, and restored after every plugin. Files go back first so no stale drop-in loads while the tables are rebuilt. Copying tables in SQL avoids depending on a mysqldump that can talk to whichever server version the run uses. If a reset fails, the results so far are reported and the run stops, since anything after it would be measured against leftovers. This replaces the per-plugin install tracking, which was only there to drive the old cleanup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 160 +++++++++++++++------ 1 file changed, 113 insertions(+), 47 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 38b9342ca8c60..4d67793f5e002 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -3,10 +3,10 @@ # Checks that plugins from the WordPress.org directory can be activated against a version of WordPress # without fataling. # -# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then removed -# again before the next one is installed, so that one broken plugin cannot mask (or break) the next. A plugin -# that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core -# refuses to activate it otherwise. +# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then the site is +# reset to the fresh install - files and database - before the next one is installed, so that one broken +# plugin cannot mask (or break) the next. A plugin that declares `Requires Plugins` has those dependencies +# installed and activated alongside it, because core refuses to activate it otherwise. # # What a real request does is what decides whether a plugin passed. The WP-CLI check runs last and only ever # adds a note, because WP-CLI loads WordPress in a way a visitor never does. @@ -300,6 +300,9 @@ run_checks() { --admin_password=password --admin_email=me@example.org --skip-email \ || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + rm -f "${WP_DIR}/wp-content/debug.log" + snapshot_site || error "The freshly installed site could not be recorded for resetting between plugins." + # The site needs to answer real requests so that fatals which only happen on a front end or admin page # load are caught. The built-in server is enough for that and needs nothing installed. # @@ -408,16 +411,9 @@ logged_fatals() { # install_plugin # -# Installs a plugin from WordPress.org and remembers it so that it is removed again once the plugin under -# test has been checked. The slug is remembered before the download is attempted so that a partial download -# is still cleaned up. Returns non zero when the download failed. +# Installs a plugin from WordPress.org. Returns non zero when the download failed. install_plugin() { - INSTALLED_SLUGS[${#INSTALLED_SLUGS[@]}]="${1}" - INSTALLED_DIRS[${#INSTALLED_DIRS[@]}]="" - - wp plugin install "${1}" --skip-plugins --skip-themes || return 1 - - INSTALLED_DIRS[${#INSTALLED_DIRS[@]} - 1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + wp plugin install "${1}" --skip-plugins --skip-themes } # is_active @@ -456,34 +452,102 @@ plugin_dependencies() { ' 2>/dev/null } -# cleanup_plugins +# Prefix of the copies of the freshly installed site's tables, which every plugin's changes are rolled back to. +BASELINE_TABLE_PREFIX="pcbaseline_" + +# snapshot_site # -# Returns the site to a clean slate by removing everything installed for the current plugin, its -# dependencies included. A plugin that fatals can take WP-CLI down with it, so every command here is allowed -# to fail and the plugin directory is removed directly as a fallback. `--skip-plugins` keeps WP-CLI from -# loading the broken plugin while cleaning up after it. -cleanup_plugins() { - if [ "${#INSTALLED_SLUGS[@]}" -gt 0 ]; then - for INDEX in "${!INSTALLED_SLUGS[@]}"; do - CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" - CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - - wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 - wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 - - if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${CLEANUP_DIR}" - fi +# Records the freshly installed site so that it can be put back exactly after each plugin: the entries at the +# root of the install, `wp-config.php` and `wp-content` as an archive, and a copy of every database table. +snapshot_site() { + BASELINE_DIR="${WORK_DIR}/baseline" + mkdir -p "${BASELINE_DIR}" || return 1 + + ( + shopt -s dotglob nullglob - rm -rf "${WP_DIR}/wp-content/plugins/${CLEANUP_SLUG}" + for ENTRY in "${WP_DIR}"/*; do + printf '%s\n' "${ENTRY##*/}" done - fi + ) > "${BASELINE_DIR}/root-entries" || return 1 + tar -C "${WP_DIR}" -cf "${BASELINE_DIR}/files.tar" wp-config.php wp-content || return 1 + + # The table copies live in the same database, which saves depending on a database client being + # installed that can talk to whatever server version the run is using. The prefix is handed over as an + # environment variable so that it never reaches PHP as source code. + export BASELINE_TABLE_PREFIX + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + + foreach ( $wpdb->get_col( "SHOW TABLES" ) as $table ) { + // Left over from an earlier run against the same database. + if ( 0 === strpos( $table, $prefix ) ) { + continue; + } + + $wpdb->query( "DROP TABLE IF EXISTS `{$prefix}{$table}`" ); + + if ( false === $wpdb->query( "CREATE TABLE `{$prefix}{$table}` LIKE `{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$prefix}{$table}` SELECT * FROM `{$table}`" ) ) { + exit( 1 ); + } + } + ' +} + +# reset_site +# +# Puts the site back the way `snapshot_site` found it. Removing the plugin directories is not enough on its +# own: plenty of plugins write drop-ins such as `advanced-cache.php`, add constants to `wp-config.php`, or +# create tables and options, and a drop-in left loading files that are gone would break every plugin tested +# after it. The files are restored first so that no drop-in is around to be loaded while the database is put +# back. +reset_site() { + ( + shopt -s dotglob nullglob + + for ENTRY in "${WP_DIR}"/*; do + grep -qxF -- "${ENTRY##*/}" "${BASELINE_DIR}/root-entries" || rm -rf "${ENTRY}" + done + ) + + rm -rf "${WP_DIR:?}/wp-content" || return 1 + tar -C "${WP_DIR}" -xf "${BASELINE_DIR}/files.tar" || return 1 + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + $baseline = array(); - INSTALLED_SLUGS=() - INSTALLED_DIRS=() + $wpdb->query( "SET FOREIGN_KEY_CHECKS = 0" ); - # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. - wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 + foreach ( $wpdb->get_results( "SHOW FULL TABLES", ARRAY_N ) as $row ) { + list( $table, $type ) = $row; + + if ( 0 === strpos( $table, $prefix ) ) { + $baseline[] = substr( $table, strlen( $prefix ) ); + } else { + $wpdb->query( ( "VIEW" === $type ? "DROP VIEW" : "DROP TABLE" ) . " `{$table}`" ); + } + } + + if ( ! $baseline ) { + exit( 1 ); + } + + foreach ( $baseline as $table ) { + if ( false === $wpdb->query( "CREATE TABLE `{$table}` LIKE `{$prefix}{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$table}` SELECT * FROM `{$prefix}{$table}`" ) ) { + exit( 1 ); + } + } + ' } # group_start / group_end @@ -504,19 +568,21 @@ group_end() { fi } +# reset_failed +# +# Stops the run when the site could not be put back after a plugin. Every result after that point would be +# measured against whatever the plugin left behind, so the results so far are reported and nothing more is +# tested. +reset_failed() { + printf '\nThe site could not be reset after %s, so no further plugins were tested.\n' "${1}" >&2 + report_results + exit 1 +} + ## # Tests each plugin in turn, writing a row per plugin to the results file. ## test_plugins() { - # The plugins directory itself must never be removed, only directories inside it. - PLUGINS_ROOT="$( wp plugin path )" - - # Everything installed for the plugin currently under test, that plugin included. The directories are - # tracked alongside the slugs because a plugin does not always unpack into a directory named after its - # slug. - INSTALLED_SLUGS=() - INSTALLED_DIRS=() - while IFS= read -r SLUG; do [ -n "${SLUG}" ] || continue @@ -534,7 +600,7 @@ test_plugins() { # it generally means a network flake or a plugin that is no longer in the directory. if ! install_plugin "${SLUG}"; then record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" - cleanup_plugins + reset_site || reset_failed "${SLUG}" group_end continue fi @@ -742,7 +808,7 @@ test_plugins() { fi record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" - cleanup_plugins + reset_site || reset_failed "${SLUG}" printf '%s: %s\n' "${SLUG}" "${STATUS}" group_end From 9b3b1d1bfc45af77445497bc9e4e890a7fac0f23 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 11:11:14 -0800 Subject: [PATCH 18/27] Build/Test Tools: Skip plugins whose PHP or WordPress requirement is unmet. The activation output was matched against "requires PHP version" and "requires WordPress version", which core does not print. Core says "does not meet minimum requirements", so a plugin declaring a newer PHP or WordPress than the run uses was recorded as FAIL instead of SKIPPED, blaming the plugin for core refusing to activate it as designed. Found by a fixture plugin that declares Requires PHP: 99.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 4d67793f5e002..3f87500883833 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -731,7 +731,7 @@ test_plugins() { # version of PHP or WordPress this run is not using, or for a dependency that is not on # WordPress.org. That is core working as designed rather than a fatal, so it is recorded # as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + *"to be installed and activated"* | *"does not meet minimum requirements"* ) STATUS="SKIPPED" REASON="Core declined to activate the plugin because its declared requirements are not met" ;; From 89698d0027fef3db14c27743e25c4ab3b6be8f7b Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 11:11:14 -0800 Subject: [PATCH 19/27] Build/Test Tools: Wait for the real database server in local plugin checks. The mysql image initialises its data directory with a temporary server that answers on the socket and then restarts. A socket ping could succeed in that window, and the next step failed with a connection refused error. The ping now goes over TCP, which only the real server listens on. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 3f87500883833..8d4234d63291e 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -171,10 +171,14 @@ run_in_docker() { "${DB_IMAGE}" > /dev/null || error "The database container could not be started." # A first run has to initialise the data directory, which takes appreciably longer than a later one. + # + # The image initialises it with a temporary server that answers on the socket and then restarts, so the + # ping goes over TCP, which only the real server listens on. A socket ping can succeed in that window and + # leave the next step to find the database gone. DB_READY="no" for _ in $( seq 1 90 ); do - if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --protocol=tcp --host=127.0.0.1 --user=root --password="${DB_PASS}" > /dev/null 2>&1; then DB_READY="yes" break fi From 2ff99b7fe94f00161cbeb64ca000d711bd1733cc Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 11:11:15 -0800 Subject: [PATCH 20/27] Build/Test Tools: Add fixture plugins that the compatibility checks must catch. Nothing showed that a broken plugin is actually caught: the pull request runs use real plugins that all pass, so a change that stopped detecting fatals would still go green. Each fixture breaks in one specific way - fatal on load, on activation, on the front end, on the login screen, after output, a late parse error, exit during activation, unmet PHP or WordPress requirements, a broken dependency, files and tables left behind for the next plugin - and expected.txt lists the result and reason each one must get. --fixtures installs a plugin from a local directory when one matches the slug, so every check still runs the same code, and --expect compares the results with the file and decides the exit code by whether they match. Weakening the activation check, the parse error check or the reset each makes the fixture run fail. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- .../fixtures/expected.txt | 27 ++++ .../fixture-broken-dependency.php | 8 ++ .../fixture-checks-for-leftovers.php | 19 +++ .../fixture-exit-on-activation.php | 13 ++ .../fixture-fatal-after-output.php | 18 +++ .../fixture-fatal-on-activation.php | 8 ++ .../fixture-fatal-on-front-end.php | 8 ++ .../fixture-fatal-on-load.php | 8 ++ .../fixture-fatal-on-login.php | 8 ++ .../fixture-healthy/fixture-healthy.php | 8 ++ .../fixture-leaves-files-behind.php | 21 ++++ .../fixture-parse-error-after-output.php | 19 +++ .../syntax-error.inc | 3 + .../fixture-requires-akismet.php | 9 ++ .../fixture-requires-broken-dependency.php | 9 ++ .../fixture-requires-newer-php.php | 9 ++ .../fixture-requires-newer-wordpress.php | 9 ++ .../fixture-wp-cli-only-fatal.php | 19 +++ tools/plugin-compatibility/test-plugins.sh | 117 +++++++++++++++++- 19 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 tools/plugin-compatibility/fixtures/expected.txt create mode 100644 tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-checks-for-leftovers/fixture-checks-for-leftovers.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-fatal-after-output/fixture-fatal-after-output.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-fatal-on-activation/fixture-fatal-on-activation.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-fatal-on-front-end/fixture-fatal-on-front-end.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-fatal-on-load/fixture-fatal-on-load.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-fatal-on-login/fixture-fatal-on-login.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-healthy/fixture-healthy.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-leaves-files-behind/fixture-leaves-files-behind.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/syntax-error.inc create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-akismet/fixture-requires-akismet.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-broken-dependency/fixture-requires-broken-dependency.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-newer-php/fixture-requires-newer-php.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-newer-wordpress/fixture-requires-newer-wordpress.php create mode 100644 tools/plugin-compatibility/fixtures/fixture-wp-cli-only-fatal/fixture-wp-cli-only-fatal.php diff --git a/tools/plugin-compatibility/fixtures/expected.txt b/tools/plugin-compatibility/fixtures/expected.txt new file mode 100644 index 0000000000000..e3acbfd6c40ff --- /dev/null +++ b/tools/plugin-compatibility/fixtures/expected.txt @@ -0,0 +1,27 @@ +# The result each fixture plugin is expected to get from test-plugins.sh, in the order they are tested. +# +# test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures --expect=tools/plugin-compatibility/fixtures/expected.txt +# +# Each fixture is a plugin built to break in one specific way, so that a change to the checks which stops a +# broken plugin from being caught fails a run instead of quietly passing everything. +# +# Format: [text the "Also active" or "Details" column must contain] + +fixture-healthy PASS +fixture-fatal-on-load FAIL The plugin could not be activated +fixture-fatal-on-activation FAIL The plugin could not be activated +fixture-exit-on-activation SKIPPED the plugin was not active afterwards +fixture-requires-newer-php SKIPPED declared requirements are not met +fixture-requires-newer-wordpress SKIPPED declared requirements are not met +fixture-fatal-on-front-end FAIL The request to / returned HTTP 500 +fixture-fatal-on-login FAIL The request to /wp-login.php returned HTTP 500 +fixture-fatal-after-output FAIL PHP Fatal error +fixture-parse-error-after-output FAIL PHP Parse error +fixture-wp-cli-only-fatal PASS WP-CLI fatals when it loads WordPress +fixture-requires-akismet PASS akismet +fixture-requires-broken-dependency SKIPPED The required plugins are not healthy on their own + +# These two must stay together and in this order: the second fatals if anything the first wrote survived the +# reset between plugins. +fixture-leaves-files-behind PASS +fixture-checks-for-leftovers PASS diff --git a/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php b/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php new file mode 100644 index 0000000000000..e4f3d6047cfd2 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php @@ -0,0 +1,8 @@ +get_var( "SHOW TABLES LIKE '{$wpdb->prefix}fixture_leftover'" ) ) { + fixture_function_that_does_not_exist(); + } + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php new file mode 100644 index 0000000000000..cd4cbbfdb6d8c --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php @@ -0,0 +1,13 @@ +query( "CREATE TABLE IF NOT EXISTS {$wpdb->prefix}fixture_leftover ( id INT )" ); + update_option( 'fixture_leftover', 'left behind' ); + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php new file mode 100644 index 0000000000000..d6a9db897cc96 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php @@ -0,0 +1,19 @@ + Port the PHP built-in server listens on. Default: 8889. --keep Leave the WordPress install in place on exit rather than removing it. Only used with --no-docker. + --fixtures= Install a plugin from / instead of WordPress.org when that directory exists. + --expect= Check each plugin's result against the one listed in , and exit 0 only when they + all match. Plugins are taken from the file when none are named. -h, --help Print this message. Examples: @@ -92,6 +97,10 @@ Examples: # Run the checks here, against a database that is already running. test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + # Check that plugins built to break are still caught. + test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures \ + --expect=tools/plugin-compatibility/fixtures/expected.txt + Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core declined to activate because its declared requirements are not met, is reported as skipped and does not fail the run. @@ -120,6 +129,8 @@ while [ "$#" -gt 0 ]; do --db-pass=* ) DB_PASS="${1#*=}" ;; --port=* ) SERVER_PORT="${1#*=}" ;; --keep ) KEEP_INSTALL="yes" ;; + --fixtures=* ) FIXTURES_DIR="${1#*=}" ;; + --expect=* ) EXPECT_FILE="${1#*=}" ;; -h | --help ) usage; exit 0 ;; -* ) usage >&2; error "Unknown option ${1}." ;; * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; @@ -127,6 +138,22 @@ while [ "$#" -gt 0 ]; do shift done +# Both are resolved now, while relative paths still mean what they meant to whoever passed them. +if [ -n "${FIXTURES_DIR}" ]; then + FIXTURES_DIR="$( cd "${FIXTURES_DIR}" > /dev/null 2>&1 && pwd )" || error "The fixtures directory does not exist." +fi + +if [ -n "${EXPECT_FILE}" ]; then + [ -f "${EXPECT_FILE}" ] || error "The expected results file does not exist." + EXPECT_DIR="$( cd "$( dirname "${EXPECT_FILE}" )" > /dev/null 2>&1 && pwd )" + EXPECT_FILE="${EXPECT_DIR%/}/$( basename "${EXPECT_FILE}" )" + + # The expected results double as the list of plugins to test, in the order they are listed. + if [ -z "$( printf '%s' "${PLUGIN_LIST}" | tr -d '[:space:],' )" ]; then + PLUGIN_LIST="$( awk '$1 !~ /^#/ && NF > 0 { print $1 }' "${EXPECT_FILE}" )" + fi +fi + # Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" @@ -192,11 +219,27 @@ run_in_docker() { printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + # Fixtures and expected results live on the host, so they are mounted read only and passed on by their + # path inside the container. + EXTRA_DOCKER_ARGS=() + EXTRA_SCRIPT_ARGS=() + + if [ -n "${FIXTURES_DIR}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${FIXTURES_DIR}:/fixtures:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --fixtures=/fixtures ) + fi + + if [ -n "${EXPECT_FILE}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${EXPECT_FILE}:/expected.txt:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --expect=/expected.txt ) + fi + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a # writable directory because WP-CLI caches its downloads there. docker run --rm --network "${DOCKER_NETWORK}" \ --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + ${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"} \ --env HOME=/tmp \ --entrypoint bash \ "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ @@ -208,6 +251,7 @@ run_in_docker() { --db-user=root \ --db-pass="${DB_PASS}" \ --port="${SERVER_PORT}" \ + ${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"} \ --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | tr '\n' ' ' )" } @@ -351,6 +395,16 @@ run_checks() { test_plugins report_results + REPORT_EXIT_CODE=$? + + # Plugins built to fail are expected to fail, so with a list of expected results it is whether they matched + # that decides the exit code. + if [ -n "${EXPECT_FILE}" ]; then + check_expectations + return + fi + + return "${REPORT_EXIT_CODE}" } # wp @@ -415,8 +469,15 @@ logged_fatals() { # install_plugin # -# Installs a plugin from WordPress.org. Returns non zero when the download failed. +# Installs a plugin from the fixtures directory when there is one by that name, and from WordPress.org +# otherwise. Returns non zero when the plugin could not be installed. install_plugin() { + if [ -n "${FIXTURES_DIR}" ] && [ -d "${FIXTURES_DIR}/${1}" ]; then + printf 'Installing %s from %s.\n' "${1}" "${FIXTURES_DIR}" + cp -R "${FIXTURES_DIR}/${1}" "${WP_DIR}/wp-content/plugins/" + return + fi + wp plugin install "${1}" --skip-plugins --skip-themes } @@ -871,6 +932,60 @@ report_results() { printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" } +## +# Compares each plugin's result with the one the expected results file lists for it, and exits non zero when +# any of them differ. +# +# Each line of the file is a slug, the expected result, and optionally some text that the "Also active" or +# "Details" column has to contain. That last part is what tells a plugin that failed for the reason it was +# built to fail apart from one that failed for some other reason. +## +check_expectations() { + MISMATCH_COUNT=0 + MATCH_COUNT=0 + + printf '\nChecking the results against %s.\n\n' "${EXPECT_FILE}" + + while read -r EXPECTED_SLUG EXPECTED_STATUS EXPECTED_TEXT; do + case "${EXPECTED_SLUG}" in + '' | '#'* ) continue ;; + esac + + ROW="$( awk -F '\t' -v slug="${EXPECTED_SLUG}" '$1 == slug { print; exit }' "${RESULTS}" )" + ACTUAL_STATUS="$( printf '%s' "${ROW}" | cut -f 3 )" + ACTUAL_TEXT="$( printf '%s' "${ROW}" | cut -f 4- | tr '\t' ' ' )" + MATCHED="yes" + + if [ -z "${ROW}" ] || [ "${ACTUAL_STATUS}" != "${EXPECTED_STATUS}" ]; then + MATCHED="no" + elif [ -n "${EXPECTED_TEXT}" ]; then + case "${ACTUAL_TEXT}" in + *"${EXPECTED_TEXT}"* ) ;; + * ) MATCHED="no" ;; + esac + fi + + if [ "${MATCHED}" = "yes" ]; then + MATCH_COUNT=$(( MATCH_COUNT + 1 )) + printf 'ok %s: %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" + else + MISMATCH_COUNT=$(( MISMATCH_COUNT + 1 )) + printf 'MISMATCH %s: expected %s %s\n got %s %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" "${EXPECTED_TEXT}" "${ACTUAL_STATUS:-no result}" "${ACTUAL_TEXT}" + fi + done < "${EXPECT_FILE}" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s of %s results matched %s.\n\n' "${MATCH_COUNT}" "$(( MATCH_COUNT + MISMATCH_COUNT ))" "$( basename "${EXPECT_FILE}" )" >> "${GITHUB_STEP_SUMMARY}" + fi + + if [ "${MISMATCH_COUNT}" -gt 0 ]; then + printf '\n%s result(s) did not match what was expected.\n' "${MISMATCH_COUNT}" + return 1 + fi + + printf '\nEvery result matched what was expected.\n' +} + if [ "${USE_DOCKER}" = "yes" ]; then run_in_docker exit $? From dc6991bedb7d196d7b2573d56c242ea79658f1d0 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 11:11:43 -0800 Subject: [PATCH 21/27] Build/Test Tools: Run the fixture plugins alongside every compatibility run. The fixtures only prove anything if they run whenever the checks change, and against the version under test, since a core change can alter what a broken plugin looks like from the outside. A fixture-tests job now runs on every event the matrix does, through the same reusable workflow with a new fixtures input, and Slack notifications wait on it too. It needs no downloads beyond WordPress itself and takes about a minute. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- .github/workflows/plugin-compatibility.yml | 22 +++++++++++++++- .../reusable-plugin-compatibility.yml | 25 ++++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index a496df94681bd..7c8b3770af1ac 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -278,13 +278,33 @@ jobs: db-type: 'mysql' db-version: '8.4' + # Tests the fixture plugins, each built to break in one specific way, and passes only when every one of them + # gets the result listed for it in tools/plugin-compatibility/fixtures/expected.txt. + # + # The real plugins above can only show that nothing broke. This is what shows that a broken plugin is still + # caught, so a change to the checks that stops catching one fails here instead of quietly passing everything. + # The fixtures are installed from the checkout, so nothing is downloaded but WordPress itself. + fixture-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Fixture plugins + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.3' + fixtures: true + db-type: 'mysql' + db-version: '8.4' + slack-notifications: name: Slack Notifications uses: ./.github/workflows/slack-notifications.yml permissions: actions: read contents: read - needs: [ build-plugin-matrix, plugin-compatibility-tests ] + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests ] if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} with: calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 003146860563c..2370725a72679 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -3,7 +3,8 @@ # activated against it without fataling. # # Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then -# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +# the site is reset to the fresh install before the next one is installed. This keeps one broken plugin from +# masking (or breaking) the next. # # The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until # that dependency is installed and active. Those dependencies are installed and activated alongside it, and the @@ -35,9 +36,15 @@ on: type: 'string' default: '8.3' plugin-slugs: - description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' - required: true + description: 'A JSON array of WordPress.org plugin slugs to test in this shard. Ignored when fixtures is true.' + required: false type: 'string' + default: '[]' + fixtures: + description: 'Test the fixture plugins in tools/plugin-compatibility/fixtures, which are built to break, and pass only when each gets the result expected for it.' + required: false + type: 'boolean' + default: false db-type: description: 'Database type. Valid types are mysql and mariadb.' required: false @@ -109,11 +116,21 @@ jobs: - name: Test each plugin in isolation env: PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + FIXTURES: ${{ inputs.fixtures }} WP_VERSION: ${{ inputs.wp-version }} DB_PORT: ${{ job.services.database.ports['3306'] }} run: | set -uo pipefail + if [ "${FIXTURES}" = "true" ]; then + SELECTION=( + --fixtures=./tools/plugin-compatibility/fixtures + --expect=./tools/plugin-compatibility/fixtures/expected.txt + ) + else + SELECTION=( --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" ) + fi + ./tools/plugin-compatibility/test-plugins.sh \ --no-docker \ --wp-version="${WP_VERSION}" \ @@ -122,4 +139,4 @@ jobs: --db-name=test_db \ --db-user=root \ --db-pass=root \ - --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" + "${SELECTION[@]}" From dfce2e4d6199f83be5a3ad732f08e411350942a5 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 13:47:49 -0800 Subject: [PATCH 22/27] Build/Test Tools: Retry plugin compatibility runs whose tests failed. The retry job only listed Slack in `needs`, and `needs.*.result` only covers listed jobs. Slack succeeds even when it is reporting a failure, so a failed matrix or fixture run never dispatched its retry. Listing the test jobs alongside Slack matches coding-standards.yml and e2e-tests.yml. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- .github/workflows/plugin-compatibility.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 7c8b3770af1ac..ab03941acddb5 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -319,7 +319,9 @@ jobs: runs-on: ubuntu-24.04 permissions: actions: write - needs: [ slack-notifications ] + # `needs` only exposes the result of the jobs listed here, and Slack succeeds whether or not the runs it + # reports on did, so the test jobs are listed too. + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests, slack-notifications ] if: | always() && github.repository == 'WordPress/wordpress-develop' && From f60ad40d0f07e3c94033c053373a789d9becf4b3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 13:48:48 -0800 Subject: [PATCH 23/27] Build/Test Tools: Stop the compatibility checks on an interrupt. The INT and TERM traps ran cleanup and then let Bash carry on, so an interrupt while waiting for the database left the loop polling a container that had just been removed, and one during the checks left them running against a stopped server and deleted working files. Interrupts now exit with 130 or 143 and cleanup runs once, from the EXIT trap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 45c35b2265ac5..76f47d9c5ecce 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -115,6 +115,15 @@ error() { exit 1 } +# exit_on_signal +# +# Exits on an interrupt rather than carrying on, so that the EXIT trap cleans up once and nothing runs against +# the server or container it removed. A trap that only cleans up hands control back to whatever was running. +exit_on_signal() { + trap 'exit 130' INT + trap 'exit 143' TERM +} + while [ "$#" -gt 0 ]; do case "${1}" in --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; @@ -186,7 +195,8 @@ run_in_docker() { # Both are removed however this exits, an interrupt included. `--volumes` takes the anonymous volume the # database image keeps its data in along with the container, which would otherwise pile up run after run. - trap 'docker rm --force --volumes "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM + trap 'docker rm --force --volumes "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT + exit_on_signal printf 'Starting %s.\n' "${DB_IMAGE}" @@ -312,7 +322,8 @@ run_checks() { SERVER_PID="" } - trap cleanup_environment EXIT INT TERM + trap cleanup_environment EXIT + exit_on_signal cd "${WP_DIR}" || error "The WordPress directory could not be entered." From e70e749944f78439d52149240e61e17952a61c8b Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 13:48:48 -0800 Subject: [PATCH 24/27] Build/Test Tools: Run a WP-CLI shell launcher rather than passing it to PHP. The helper always ran `php `, which works for the phar but prints a shell launcher such as a Composer install's `bin/wp` as text and exits zero. Setup then appeared to succeed without WordPress being installed. A `wp` whose shebang does not name PHP is now run directly, with the memory limit passed through the `WP_CLI_PHP_ARGS` the launcher reads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 76f47d9c5ecce..098f387555221 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -278,6 +278,14 @@ run_checks() { [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + # A phar or PHP entry point either has no shebang or one naming PHP. Any other shebang is a launcher. + WP_CLI_IS_LAUNCHER="no" + + case "$( head -n 1 "${WP_CLI_BIN}" 2>/dev/null | tr -d '\000' )" in + '#!'*php* ) ;; + '#!'* ) WP_CLI_IS_LAUNCHER="yes" ;; + esac + WORK_DIR="$( mktemp -d )" RESULTS="${WORK_DIR}/results.tsv" RESPONSE_BODY="${WORK_DIR}/response.html" @@ -421,10 +429,15 @@ run_checks() { # wp # # Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock -# php.ini allows, and WP-CLI reads its own `WP_CLI_PHP_ARGS` only when it is installed as a wrapper script -# rather than as the phar most environments carry. +# php.ini allows. The phar most environments carry is handed to PHP with the limit set, while a shell +# launcher, such as the `bin/wp` of a Composer install, is run as it is and passed the limit through the +# `WP_CLI_PHP_ARGS` it reads. Handing a shell script to PHP would print it rather than run it. wp() { - php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" + if [ "${WP_CLI_IS_LAUNCHER}" = "yes" ]; then + WP_CLI_PHP_ARGS="-d memory_limit=-1${WP_CLI_PHP_ARGS:+ ${WP_CLI_PHP_ARGS}}" "${WP_CLI_BIN}" "$@" + else + php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" + fi } # record From 8ce87fe26c79a41513a0f9c103954c13d937dc68 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 13:48:48 -0800 Subject: [PATCH 25/27] Build/Test Tools: Match core's validation of dependency slugs. The `Requires Plugins` parser accepted any run of letters, digits and hyphens, so an entry such as `foo--bar` or `-foo`, which core ignores, could make a plugin skip for a dependency that is not available. It now uses the pattern from WP_Plugin_Dependencies::sanitize_dependency_slugs(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/test-plugins.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 098f387555221..8c14c598befc0 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -533,8 +533,9 @@ plugin_dependencies() { $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { - // Core only accepts a WordPress.org slug here, so anything else is ignored. - if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + // The same pattern as WP_Plugin_Dependencies::sanitize_dependency_slugs(). Core ignores anything + // else, so a malformed entry must not make the plugin wait on a dependency core never asks for. + if ( preg_match( "/^[a-z0-9]+(-[a-z0-9]+)*$/", trim( $dependency ) ) ) { echo trim( $dependency ), "\n"; } } From b44e9a3ec9818000a2f8e79ed1bee8885f120bd3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 2 Nov 2026 13:49:31 -0800 Subject: [PATCH 26/27] Build/Test Tools: Skip plugins needing both a newer WordPress and PHP. Core words its refusal "do not meet minimum requirements" when both versions are too old, and the check only matched the singular "does not meet" it uses for one. Such a plugin was reported as a failure and failed its shard, though nothing fataled. The fixture covers the combined case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/fixtures/expected.txt | 1 + .../fixture-requires-newer-both.php | 10 ++++++++++ tools/plugin-compatibility/test-plugins.sh | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-newer-both/fixture-requires-newer-both.php diff --git a/tools/plugin-compatibility/fixtures/expected.txt b/tools/plugin-compatibility/fixtures/expected.txt index e3acbfd6c40ff..00731445e24c5 100644 --- a/tools/plugin-compatibility/fixtures/expected.txt +++ b/tools/plugin-compatibility/fixtures/expected.txt @@ -13,6 +13,7 @@ fixture-fatal-on-activation FAIL The plugin could not be activated fixture-exit-on-activation SKIPPED the plugin was not active afterwards fixture-requires-newer-php SKIPPED declared requirements are not met fixture-requires-newer-wordpress SKIPPED declared requirements are not met +fixture-requires-newer-both SKIPPED declared requirements are not met fixture-fatal-on-front-end FAIL The request to / returned HTTP 500 fixture-fatal-on-login FAIL The request to /wp-login.php returned HTTP 500 fixture-fatal-after-output FAIL PHP Fatal error diff --git a/tools/plugin-compatibility/fixtures/fixture-requires-newer-both/fixture-requires-newer-both.php b/tools/plugin-compatibility/fixtures/fixture-requires-newer-both/fixture-requires-newer-both.php new file mode 100644 index 0000000000000..75e641a2af2c4 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-requires-newer-both/fixture-requires-newer-both.php @@ -0,0 +1,10 @@ + Date: Mon, 2 Nov 2026 13:59:17 -0800 Subject: [PATCH 27/27] Build/Test Tools: Skip plugins whose dependencies WP-CLI cannot load. The plugin under test is activated through WP-CLI, which loads its active dependencies first. A dependency that is healthy over HTTP but fatals under WP-CLI, which the final check already treats as a note rather than a failure, stopped that activation and the plugin under test was reported as failing. WP-CLI is now checked to load with the dependencies active before activating the plugin under test, and the plugin is skipped when it cannot. The fixture pairs a plugin with the existing WP-CLI only fatal fixture as its dependency. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018vwYjTH76XPsP8BDwweFbg --- tools/plugin-compatibility/fixtures/expected.txt | 1 + .../fixture-requires-wp-cli-only-fatal.php | 9 +++++++++ tools/plugin-compatibility/test-plugins.sh | 13 +++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 tools/plugin-compatibility/fixtures/fixture-requires-wp-cli-only-fatal/fixture-requires-wp-cli-only-fatal.php diff --git a/tools/plugin-compatibility/fixtures/expected.txt b/tools/plugin-compatibility/fixtures/expected.txt index 00731445e24c5..aad4bcb02a348 100644 --- a/tools/plugin-compatibility/fixtures/expected.txt +++ b/tools/plugin-compatibility/fixtures/expected.txt @@ -21,6 +21,7 @@ fixture-parse-error-after-output FAIL PHP Parse error fixture-wp-cli-only-fatal PASS WP-CLI fatals when it loads WordPress fixture-requires-akismet PASS akismet fixture-requires-broken-dependency SKIPPED The required plugins are not healthy on their own +fixture-requires-wp-cli-only-fatal SKIPPED WP-CLI cannot load WordPress with the required plugins active # These two must stay together and in this order: the second fatals if anything the first wrote survived the # reset between plugins. diff --git a/tools/plugin-compatibility/fixtures/fixture-requires-wp-cli-only-fatal/fixture-requires-wp-cli-only-fatal.php b/tools/plugin-compatibility/fixtures/fixture-requires-wp-cli-only-fatal/fixture-requires-wp-cli-only-fatal.php new file mode 100644 index 0000000000000..d728e1e33cc54 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-requires-wp-cli-only-fatal/fixture-requires-wp-cli-only-fatal.php @@ -0,0 +1,9 @@ +&1 )" + + if [ "${DEPENDENCY_EVAL_OUTPUT#*loaded-ok}" = "${DEPENDENCY_EVAL_OUTPUT}" ]; then + printf '%s\n' "${DEPENDENCY_EVAL_OUTPUT}" + STATUS="SKIPPED" + REASON="WP-CLI cannot load WordPress with the required plugins active, so the plugin cannot be activated" + fi + fi + # Anything else the dependencies logged on their way up is not the responsibility of the plugin # under test, so the log starts empty again here. rm -f "${WP_DIR}/wp-content/debug.log"