diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..ab03941acddb5 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,350 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. +# +# A dispatched run can also name the plugins to test instead of taking them from the directory, which is how to +# check a single plugin against a release candidate, or to reproduce a failure from an earlier run without +# waiting for the other plugins in its shard. +# +# The checks each shard runs live in `tools/plugin-compatibility/test-plugins.sh`. Running that script from a +# terminal runs exactly what a shard here runs, which is the quickest way to reproduce a result from a run +# without waiting on Actions. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' + type: string + default: 'nightly' + plugin-count: + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250. Ignored when plugin-slugs is given.' + type: string + default: '100' + plugin-slugs: + description: 'Optional. Test these WordPress.org plugin slugs instead of the most popular ones. Accepts a comma or space separated list, eg. "woocommerce, classic-editor".' + type: string + default: '' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + # + # Dispatched runs are grouped by run instead, so that testing one plugin list does not cancel a run someone + # else started with a different one against the same version of WordPress. + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. A dispatched run can name the plugins to test + # instead, in which case the directory is not queried at all. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins, or reads the plugin-slugs input. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + # Dispatched runs are allowed anywhere so that this can be run from a fork, which is the only way for + # someone without write access to the repository to try it against a plugin of their choosing. + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Build the list of plugins to test + id: plugin-shards + env: + # An explicit list of slugs to test in place of the most popular ones. Empty for everything but a + # dispatched run that filled the input in. + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' + run: | + set -euo pipefail + + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + + if [ -n "${PLUGIN_SLUGS}" ]; then + # A named list replaces the popularity query entirely. Commas and whitespace both separate slugs, + # so a list can be pasted in however it was written down. + printf '%s' "${PLUGIN_SLUGS}" | tr ',[:space:]' '\n' > "${RAW_SLUGS}" + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + + # A value outside the character set the directory uses is a typo worth stopping for now, rather + # than a plugin reported as impossible to download several minutes from now. + if INVALID_SLUGS="$( grep -Ev '^[a-z0-9-]+$' "${DEDUPED_SLUGS}" )"; then + printf 'The plugin-slugs input contains values that are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" + exit 1 + fi + + PLUGIN_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + if [ "${PLUGIN_COUNT}" -lt 1 ]; then + printf 'The plugin-slugs input did not contain any plugin slugs.\n' + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-slugs input names %s plugins, more than the %s this workflow will test in one run.\n' "${PLUGIN_COUNT}" "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + printf 'Testing the %s plugin(s) named in the plugin-slugs input.\n' "${PLUGIN_COUNT}" + else + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + fi + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + # Tests the fixture plugins, each built to break in one specific way, and passes only when every one of them + # gets the result listed for it in tools/plugin-compatibility/fixtures/expected.txt. + # + # The real plugins above can only show that nothing broke. This is what shows that a broken plugin is still + # caught, so a change to the checks that stops catching one fails here instead of quietly passing everything. + # The fixtures are installed from the checkout, so nothing is downloaded but WordPress itself. + fixture-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Fixture plugins + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.3' + fixtures: true + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + # `needs` only exposes the result of the jobs listed here, and Slack succeeds whether or not the runs it + # reports on did, so the test jobs are listed too. + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests, slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..2370725a72679 --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,142 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# the site is reset to the fresh install before the next one is installed. This keeps one broken plugin from +# masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. +# +# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that +# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top +# of that script. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard. Ignored when fixtures is true.' + required: false + type: 'string' + default: '[]' + fixtures: + description: 'Test the fixture plugins in tools/plugin-compatibility/fixtures, which are built to break, and pass only when each gets the result expected for it.' + required: false + type: 'boolean' + default: false + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Checks out the repository, for the script that runs the checks. + # - Sets up PHP and WP-CLI. + # - Runs the checks against every plugin in the shard. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 + + services: + database: + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the + # containers the script provisions when it is run from a terminal. + # + # WordPress is installed outside the workspace so that the checkout above is not part of the site under + # test. + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + FIXTURES: ${{ inputs.fixtures }} + WP_VERSION: ${{ inputs.wp-version }} + DB_PORT: ${{ job.services.database.ports['3306'] }} + run: | + set -uo pipefail + + if [ "${FIXTURES}" = "true" ]; then + SELECTION=( + --fixtures=./tools/plugin-compatibility/fixtures + --expect=./tools/plugin-compatibility/fixtures/expected.txt + ) + else + SELECTION=( --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" ) + fi + + ./tools/plugin-compatibility/test-plugins.sh \ + --no-docker \ + --wp-version="${WP_VERSION}" \ + --dir="${RUNNER_TEMP}/wordpress" \ + --db-host="127.0.0.1:${DB_PORT}" \ + --db-name=test_db \ + --db-user=root \ + --db-pass=root \ + "${SELECTION[@]}" diff --git a/tools/plugin-compatibility/fixtures/expected.txt b/tools/plugin-compatibility/fixtures/expected.txt new file mode 100644 index 0000000000000..aad4bcb02a348 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/expected.txt @@ -0,0 +1,29 @@ +# The result each fixture plugin is expected to get from test-plugins.sh, in the order they are tested. +# +# test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures --expect=tools/plugin-compatibility/fixtures/expected.txt +# +# Each fixture is a plugin built to break in one specific way, so that a change to the checks which stops a +# broken plugin from being caught fails a run instead of quietly passing everything. +# +# Format: [text the "Also active" or "Details" column must contain] + +fixture-healthy PASS +fixture-fatal-on-load FAIL The plugin could not be activated +fixture-fatal-on-activation FAIL The plugin could not be activated +fixture-exit-on-activation SKIPPED the plugin was not active afterwards +fixture-requires-newer-php SKIPPED declared requirements are not met +fixture-requires-newer-wordpress SKIPPED declared requirements are not met +fixture-requires-newer-both SKIPPED declared requirements are not met +fixture-fatal-on-front-end FAIL The request to / returned HTTP 500 +fixture-fatal-on-login FAIL The request to /wp-login.php returned HTTP 500 +fixture-fatal-after-output FAIL PHP Fatal error +fixture-parse-error-after-output FAIL PHP Parse error +fixture-wp-cli-only-fatal PASS WP-CLI fatals when it loads WordPress +fixture-requires-akismet PASS akismet +fixture-requires-broken-dependency SKIPPED The required plugins are not healthy on their own +fixture-requires-wp-cli-only-fatal SKIPPED WP-CLI cannot load WordPress with the required plugins active + +# These two must stay together and in this order: the second fatals if anything the first wrote survived the +# reset between plugins. +fixture-leaves-files-behind PASS +fixture-checks-for-leftovers PASS diff --git a/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php b/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php new file mode 100644 index 0000000000000..e4f3d6047cfd2 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php @@ -0,0 +1,8 @@ +get_var( "SHOW TABLES LIKE '{$wpdb->prefix}fixture_leftover'" ) ) { + fixture_function_that_does_not_exist(); + } + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php new file mode 100644 index 0000000000000..cd4cbbfdb6d8c --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php @@ -0,0 +1,13 @@ +query( "CREATE TABLE IF NOT EXISTS {$wpdb->prefix}fixture_leftover ( id INT )" ); + update_option( 'fixture_leftover', 'left behind' ); + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php new file mode 100644 index 0000000000000..d6a9db897cc96 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php @@ -0,0 +1,19 @@ + /dev/null 2>&1 && pwd )/$( basename "${BASH_SOURCE[0]}" )" + +# Defaults. Every one of these can be overridden with the matching option. +WP_VERSION="nightly" +PLUGIN_LIST="" +USE_DOCKER="yes" +# The official WP-CLI image carries PHP, WP-CLI, `mysqli` and curl, which is everything the checks need. +PHP_IMAGE="wordpress:cli-php8.3" +# Matches the database the workflow runs against by default. +DB_IMAGE="mysql:8.4" +WP_DIR="" +DB_HOST="127.0.0.1" +DB_NAME="test_db" +DB_USER="root" +DB_PASS="password" +SERVER_PORT="8889" +KEEP_INSTALL="no" +FIXTURES_DIR="" +EXPECT_FILE="" + +# How many levels of `Requires Plugins` to follow. Chains in the directory are one level deep in practice, so +# the bound is only here to stop a circular declaration from looping forever. +MAX_DEPENDENCY_DEPTH=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-plugins.sh [options] [slug...] + +Checks that WordPress.org plugins can be activated against a version of WordPress without fataling. + +Options: + --plugins= Plugin slugs to test, separated by commas or spaces. Slugs can also be passed as + positional arguments. + --wp-version= Version of WordPress to test against. Accepts "latest", "nightly", or a version + number such as a beta or RC. Default: nightly. + --php-image= Container image the checks run in. Default: wordpress:cli-php8.3. + --db-image= Container image the database runs in. Default: mysql:8.4. + --no-docker Run the checks in the current environment rather than in a container. Requires PHP, + WP-CLI, curl and a database that is already running. + --dir= Where to install WordPress. Only used with --no-docker. Default: a temporary + directory, which is removed again on exit. + --db-host= Database host, with an optional port. Only used with --no-docker. Default: 127.0.0.1. + --db-name= Database name. Only used with --no-docker. Default: test_db. + --db-user= Database user. Only used with --no-docker. Default: root. + --db-pass= Database password. Only used with --no-docker. Default: password. + --port= Port the PHP built-in server listens on. Default: 8889. + --keep Leave the WordPress install in place on exit rather than removing it. Only used with + --no-docker. + --fixtures= Install a plugin from / instead of WordPress.org when that directory exists. + --expect= Check each plugin's result against the one listed in , and exit 0 only when they + all match. Plugins are taken from the file when none are named. + -h, --help Print this message. + +Examples: + # Check two plugins against nightly, provisioning everything in containers. + test-plugins.sh --wp-version=nightly woocommerce classic-editor + + # Reproduce a failure from a workflow run against the release it was seen on. + test-plugins.sh --wp-version=7.1-RC1 eps-301-redirects + + # Run the checks here, against a database that is already running. + test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + + # Check that plugins built to break are still caught. + test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures \ + --expect=tools/plugin-compatibility/fixtures/expected.txt + +Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core +declined to activate because its declared requirements are not met, is reported as skipped and does not +fail the run. +USAGE_EOF +} + +# error +# +# Prints a message to stderr and stops. +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +# exit_on_signal +# +# Exits on an interrupt rather than carrying on, so that the EXIT trap cleans up once and nothing runs against +# the server or container it removed. A trap that only cleans up hands control back to whatever was running. +exit_on_signal() { + trap 'exit 130' INT + trap 'exit 143' TERM +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --php-image=* ) PHP_IMAGE="${1#*=}" ;; + --db-image=* ) DB_IMAGE="${1#*=}" ;; + --no-docker ) USE_DOCKER="no" ;; + --dir=* ) WP_DIR="${1#*=}" ;; + --db-host=* ) DB_HOST="${1#*=}" ;; + --db-name=* ) DB_NAME="${1#*=}" ;; + --db-user=* ) DB_USER="${1#*=}" ;; + --db-pass=* ) DB_PASS="${1#*=}" ;; + --port=* ) SERVER_PORT="${1#*=}" ;; + --keep ) KEEP_INSTALL="yes" ;; + --fixtures=* ) FIXTURES_DIR="${1#*=}" ;; + --expect=* ) EXPECT_FILE="${1#*=}" ;; + -h | --help ) usage; exit 0 ;; + -* ) usage >&2; error "Unknown option ${1}." ;; + * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; + esac + shift +done + +# Both are resolved now, while relative paths still mean what they meant to whoever passed them. +if [ -n "${FIXTURES_DIR}" ]; then + FIXTURES_DIR="$( cd "${FIXTURES_DIR}" > /dev/null 2>&1 && pwd )" || error "The fixtures directory does not exist." +fi + +if [ -n "${EXPECT_FILE}" ]; then + [ -f "${EXPECT_FILE}" ] || error "The expected results file does not exist." + EXPECT_DIR="$( cd "$( dirname "${EXPECT_FILE}" )" > /dev/null 2>&1 && pwd )" + EXPECT_FILE="${EXPECT_DIR%/}/$( basename "${EXPECT_FILE}" )" + + # The expected results double as the list of plugins to test, in the order they are listed. + if [ -z "$( printf '%s' "${PLUGIN_LIST}" | tr -d '[:space:],' )" ]; then + PLUGIN_LIST="$( awk '$1 !~ /^#/ && NF > 0 { print $1 }' "${EXPECT_FILE}" )" + fi +fi + +# Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. +PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" + +if [ -z "${PLUGIN_SLUGS}" ]; then + usage >&2 + error "No plugins were named." +fi + +# A value outside the character set the directory uses is a typo worth stopping for, rather than a plugin +# reported as impossible to download several minutes from now. +INVALID_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | grep -Ev '^[a-z0-9-]+$' )" + +if [ -n "${INVALID_SLUGS}" ]; then + printf 'These values are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" >&2 + exit 1 +fi + +## +# Provisions a database and a PHP, and runs this same script inside them. +# +# The checks themselves are identical either way: the container is handed this file and runs it with +# `--no-docker`, so there is only one copy of the logic that decides whether a plugin passed. +## +run_in_docker() { + command -v docker > /dev/null 2>&1 || error "Docker is required, or pass --no-docker to run the checks here." + + # Everything is named after the process so that two runs at once do not collide. + DOCKER_NETWORK="plugin-compatibility-$$" + DB_CONTAINER="plugin-compatibility-db-$$" + + # Both are removed however this exits, an interrupt included. `--volumes` takes the anonymous volume the + # database image keeps its data in along with the container, which would otherwise pile up run after run. + trap 'docker rm --force --volumes "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT + exit_on_signal + + printf 'Starting %s.\n' "${DB_IMAGE}" + + docker network create "${DOCKER_NETWORK}" > /dev/null || error "The Docker network could not be created." + + docker run --detach --name "${DB_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --env MYSQL_ROOT_PASSWORD="${DB_PASS}" \ + --env MYSQL_DATABASE="${DB_NAME}" \ + "${DB_IMAGE}" > /dev/null || error "The database container could not be started." + + # A first run has to initialise the data directory, which takes appreciably longer than a later one. + # + # The image initialises it with a temporary server that answers on the socket and then restarts, so the + # ping goes over TCP, which only the real server listens on. A socket ping can succeed in that window and + # leave the next step to find the database gone. + DB_READY="no" + + for _ in $( seq 1 90 ); do + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --protocol=tcp --host=127.0.0.1 --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + DB_READY="yes" + break + fi + sleep 2 + done + + if [ "${DB_READY}" != "yes" ]; then + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 20 + error "The database did not become available." + fi + + printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + + # Fixtures and expected results live on the host, so they are mounted read only and passed on by their + # path inside the container. + EXTRA_DOCKER_ARGS=() + EXTRA_SCRIPT_ARGS=() + + if [ -n "${FIXTURES_DIR}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${FIXTURES_DIR}:/fixtures:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --fixtures=/fixtures ) + fi + + if [ -n "${EXPECT_FILE}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${EXPECT_FILE}:/expected.txt:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --expect=/expected.txt ) + fi + + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot + # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a + # writable directory because WP-CLI caches its downloads there. + docker run --rm --network "${DOCKER_NETWORK}" \ + --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + ${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"} \ + --env HOME=/tmp \ + --entrypoint bash \ + "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ + --no-docker \ + --dir=/tmp/wordpress \ + --wp-version="${WP_VERSION}" \ + --db-host="${DB_CONTAINER}" \ + --db-name="${DB_NAME}" \ + --db-user=root \ + --db-pass="${DB_PASS}" \ + --port="${SERVER_PORT}" \ + ${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"} \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | tr '\n' ' ' )" +} + +## +# Installs WordPress, starts a web server, and checks each plugin against it. +## +run_checks() { + for REQUIRED_COMMAND in php curl; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required to run the checks without Docker." + done + + # `type -P` searches the path rather than resolving to the `wp` wrapper function defined below. + WP_CLI_BIN="$( type -P wp )" + + [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + + # A phar or PHP entry point either has no shebang or one naming PHP. Any other shebang is a launcher. + WP_CLI_IS_LAUNCHER="no" + + case "$( head -n 1 "${WP_CLI_BIN}" 2>/dev/null | tr -d '\000' )" in + '#!'*php* ) ;; + '#!'* ) WP_CLI_IS_LAUNCHER="yes" ;; + esac + + WORK_DIR="$( mktemp -d )" + RESULTS="${WORK_DIR}/results.tsv" + RESPONSE_BODY="${WORK_DIR}/response.html" + SERVER_LOG="${WORK_DIR}/php-server.log" + SERVER_PID="" + : > "${RESULTS}" + + if [ -z "${WP_DIR}" ]; then + WP_DIR="${WORK_DIR}/wordpress" + fi + + mkdir -p "${WP_DIR}" || error "The WordPress directory could not be created." + + # The server is stopped and the temporary files are removed however this exits. The WordPress install is + # only removed when this script created it, so that `--dir` never deletes a directory it was handed. + cleanup_environment() { + if [ -n "${SERVER_PID}" ]; then + stop_server + fi + + if [ "${KEEP_INSTALL}" = "yes" ]; then + printf '\nThe WordPress install was left in %s.\n' "${WP_DIR}" + else + rm -rf "${WORK_DIR}" + fi + } + + # The built-in server forks its workers, and they keep answering on the port after the process that + # started them is gone. The server runs in its own process group (see below) so that the whole group can + # be stopped at once, and this waits for the last of them to exit so that a run which follows straight + # after finds the port free. + stop_server() { + kill -TERM -- "-${SERVER_PID}" > /dev/null 2>&1 + + for _ in $( seq 1 50 ); do + kill -0 -- "-${SERVER_PID}" > /dev/null 2>&1 || break + sleep 0.1 + done + + kill -KILL -- "-${SERVER_PID}" > /dev/null 2>&1 + wait "${SERVER_PID}" > /dev/null 2>&1 + SERVER_PID="" + } + + trap cleanup_environment EXIT + exit_on_signal + + cd "${WP_DIR}" || error "The WordPress directory could not be entered." + + # A relative `--dir` would be resolved against the wrong directory from here on. + WP_DIR="$( pwd )" + + SITE_URL="http://127.0.0.1:${SERVER_PORT}" + + printf 'Downloading WordPress %s.\n' "${WP_VERSION}" + wp core download --version="${WP_VERSION}" || error "WordPress ${WP_VERSION} could not be downloaded." + + wp config create --dbname="${DB_NAME}" --dbuser="${DB_USER}" --dbpass="${DB_PASS}" --dbhost="${DB_HOST}" \ + || error "wp-config.php could not be written." + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed + # by recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that + # spawned it, and the two deadlock until curl gives up. + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email \ + || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + + rm -f "${WP_DIR}/wp-content/debug.log" + snapshot_site || error "The freshly installed site could not be recorded for resetting between plugins." + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + # + # Job control is switched on just long enough to start the server, because that is what gives a + # background job a process group of its own, whose ID is the PID of the server. + set -m + PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + set +m + + # Any server already listening on the port would answer a request for a WordPress page just as well, and + # every check after this would then be made against that site instead. So readiness means this server is + # still running and is serving a file that only exists in this install. + READY_FILE="plugin-compatibility-ready-$$-${RANDOM}.txt" + READY_TOKEN="$$-${RANDOM}-$( date +%s )" + printf '%s' "${READY_TOKEN}" > "${WP_DIR}/${READY_FILE}" + + SERVER_READY="no" + + for _ in $( seq 1 30 ); do + kill -0 "${SERVER_PID}" > /dev/null 2>&1 || break + + if [ "$( curl -sf "${SITE_URL}/${READY_FILE}" 2>/dev/null )" = "${READY_TOKEN}" ]; then + SERVER_READY="yes" + break + fi + sleep 1 + done + + rm -f "${WP_DIR:?}/${READY_FILE}" + + if [ "${SERVER_READY}" != "yes" ]; then + cat "${SERVER_LOG}" + error "The PHP built-in server did not start on port ${SERVER_PORT}, or something else is already listening on it." + fi + + printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" + + test_plugins + report_results + REPORT_EXIT_CODE=$? + + # Plugins built to fail are expected to fail, so with a list of expected results it is whether they matched + # that decides the exit code. + if [ -n "${EXPECT_FILE}" ]; then + check_expectations + return + fi + + return "${REPORT_EXIT_CODE}" +} + +# wp +# +# Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock +# php.ini allows. The phar most environments carry is handed to PHP with the limit set, while a shell +# launcher, such as the `bin/wp` of a Composer install, is run as it is and passed the limit through the +# `WP_CLI_PHP_ARGS` it reads. Handing a shell script to PHP would print it rather than run it. +wp() { + if [ "${WP_CLI_IS_LAUNCHER}" = "yes" ]; then + WP_CLI_PHP_ARGS="-d memory_limit=-1${WP_CLI_PHP_ARGS:+ ${WP_CLI_PHP_ARGS}}" "${WP_CLI_BIN}" "$@" + else + php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" + fi +} + +# record +# +# Appends one tab separated row to the results file. The reason is flattened so that it cannot break the +# markdown table that is generated from these rows later on. +record() { + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" +} + +# check_url +# +# Requests a path on the test site and prints a reason to stdout when the response looks broken. Prints +# nothing when the request looks healthy. +check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense status + # like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for + # itself. + if grep -qiE '(Fatal|Parse) error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi +} + +# logged_fatals +# +# Prints the fatal and parse errors in the debug log, one per line, and returns non zero when there are none. +# A parse error in a file included after output has started ends the request just as surely as a fatal does, +# and still answers with a 200. +logged_fatals() { + [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -E 'PHP (Fatal|Parse) error' "${WP_DIR}/wp-content/debug.log" +} + +# install_plugin +# +# Installs a plugin from the fixtures directory when there is one by that name, and from WordPress.org +# otherwise. Returns non zero when the plugin could not be installed. +install_plugin() { + if [ -n "${FIXTURES_DIR}" ] && [ -d "${FIXTURES_DIR}/${1}" ]; then + printf 'Installing %s from %s.\n' "${1}" "${FIXTURES_DIR}" + cp -R "${FIXTURES_DIR}/${1}" "${WP_DIR}/wp-content/plugins/" + return + fi + + wp plugin install "${1}" --skip-plugins --skip-themes +} + +# is_active +# +# Returns zero when core has recorded the plugin as active. +is_active() { + wp plugin is-active "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 +} + +# plugin_dependencies +# +# Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header +# parser is used so that the same rules apply here as when core decides whether a plugin's dependencies are +# met. +plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + export PLUGIN_FILE + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // The same pattern as WP_Plugin_Dependencies::sanitize_dependency_slugs(). Core ignores anything + // else, so a malformed entry must not make the plugin wait on a dependency core never asks for. + if ( preg_match( "/^[a-z0-9]+(-[a-z0-9]+)*$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null +} + +# Prefix of the copies of the freshly installed site's tables, which every plugin's changes are rolled back to. +BASELINE_TABLE_PREFIX="pcbaseline_" + +# snapshot_site +# +# Records the freshly installed site so that it can be put back exactly after each plugin: the entries at the +# root of the install, `wp-config.php` and `wp-content` as an archive, and a copy of every database table. +snapshot_site() { + BASELINE_DIR="${WORK_DIR}/baseline" + mkdir -p "${BASELINE_DIR}" || return 1 + + ( + shopt -s dotglob nullglob + + for ENTRY in "${WP_DIR}"/*; do + printf '%s\n' "${ENTRY##*/}" + done + ) > "${BASELINE_DIR}/root-entries" || return 1 + tar -C "${WP_DIR}" -cf "${BASELINE_DIR}/files.tar" wp-config.php wp-content || return 1 + + # The table copies live in the same database, which saves depending on a database client being + # installed that can talk to whatever server version the run is using. The prefix is handed over as an + # environment variable so that it never reaches PHP as source code. + export BASELINE_TABLE_PREFIX + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + + foreach ( $wpdb->get_col( "SHOW TABLES" ) as $table ) { + // Left over from an earlier run against the same database. + if ( 0 === strpos( $table, $prefix ) ) { + continue; + } + + $wpdb->query( "DROP TABLE IF EXISTS `{$prefix}{$table}`" ); + + if ( false === $wpdb->query( "CREATE TABLE `{$prefix}{$table}` LIKE `{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$prefix}{$table}` SELECT * FROM `{$table}`" ) ) { + exit( 1 ); + } + } + ' +} + +# reset_site +# +# Puts the site back the way `snapshot_site` found it. Removing the plugin directories is not enough on its +# own: plenty of plugins write drop-ins such as `advanced-cache.php`, add constants to `wp-config.php`, or +# create tables and options, and a drop-in left loading files that are gone would break every plugin tested +# after it. The files are restored first so that no drop-in is around to be loaded while the database is put +# back. +reset_site() { + ( + shopt -s dotglob nullglob + + for ENTRY in "${WP_DIR}"/*; do + grep -qxF -- "${ENTRY##*/}" "${BASELINE_DIR}/root-entries" || rm -rf "${ENTRY}" + done + ) + + rm -rf "${WP_DIR:?}/wp-content" || return 1 + tar -C "${WP_DIR}" -xf "${BASELINE_DIR}/files.tar" || return 1 + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + $baseline = array(); + + $wpdb->query( "SET FOREIGN_KEY_CHECKS = 0" ); + + foreach ( $wpdb->get_results( "SHOW FULL TABLES", ARRAY_N ) as $row ) { + list( $table, $type ) = $row; + + if ( 0 === strpos( $table, $prefix ) ) { + $baseline[] = substr( $table, strlen( $prefix ) ); + } else { + $wpdb->query( ( "VIEW" === $type ? "DROP VIEW" : "DROP TABLE" ) . " `{$table}`" ); + } + } + + if ( ! $baseline ) { + exit( 1 ); + } + + foreach ( $baseline as $table ) { + if ( false === $wpdb->query( "CREATE TABLE `{$table}` LIKE `{$prefix}{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$table}` SELECT * FROM `{$prefix}{$table}`" ) ) { + exit( 1 ); + } + } + ' +} + +# group_start / group_end +# +# Collapses a plugin's output in the workflow log. The markers mean nothing to a terminal, so they are only +# printed when this is running in Actions. +group_start() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::group::%s\n' "${1}" + else + printf '\n----- %s -----\n' "${1}" + fi +} + +group_end() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::endgroup::\n' + fi +} + +# reset_failed +# +# Stops the run when the site could not be put back after a plugin. Every result after that point would be +# measured against whatever the plugin left behind, so the results so far are reported and nothing more is +# tested. +reset_failed() { + printf '\nThe site could not be reset after %s, so no further plugins were tested.\n' "${1}" >&2 + report_results + exit 1 +} + +## +# Tests each plugin in turn, writing a row per plugin to the results file. +## +test_plugins() { + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + group_start "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + DEPENDENCIES=() + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f "${WP_DIR}/wp-content/debug.log" + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, since + # it generally means a network flake or a plugin that is no longer in the directory. + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + reset_site || reset_failed "${SLUG}" + group_end + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a plugin + # whose dependencies are missing, so without this every WooCommerce extension - a large slice of the + # most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is installed + # too. Each one is resolved only once, which covers a plugin named twice in the same tree as well as a + # declaration pointing back at the plugin under test. A dependency that is already installed, such as + # the Akismet that ships with core, is not downloaded again but is still activated and followed. + PENDING=() + RESOLVED=$'\n'"${SLUG}"$'\n' + + while IFS= read -r DEPENDENCY; do + [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" + done < <( plugin_dependencies "${SLUG}" ) + + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + case "${RESOLVED}" in + *$'\n'"${DEPENDENCY}"$'\n'* ) continue ;; + esac + + RESOLVED="${RESOLVED}${DEPENDENCY}"$'\n' + + if ! wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + fi + + DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" + + while IFS= read -r NESTED_DEPENDENCY; do + [ -n "${NESTED_DEPENDENCY}" ] && NEXT[${#NEXT[@]}]="${NESTED_DEPENDENCY}" + done < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( ${NEXT[@]+"${NEXT[@]}"} ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies. Core will not activate a plugin ahead of its own requirements, + # and the order they were discovered in says nothing about the order they depend on each other in - + # a plugin can require `base, extension` where `extension` requires `base` as well. So they are + # activated in passes: each pass activates whatever it can, and whatever core declined is tried + # again on the next. A pass that activates nothing means the rest can never be activated, whether + # because one of them is broken or because they require each other. + # + # A successful exit is not taken as proof of activation. A plugin that calls `exit` while activating + # ends WP-CLI with a zero status before core has recorded it as active. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + INACTIVE=( "${DEPENDENCIES[@]}" ) + + while [ "${#INACTIVE[@]}" -gt 0 ]; do + STILL_INACTIVE=() + + for DEPENDENCY in "${INACTIVE[@]}"; do + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if ! is_active "${DEPENDENCY}"; then + STILL_INACTIVE[${#STILL_INACTIVE[@]}]="${DEPENDENCY}" + fi + done + + if [ "${#STILL_INACTIVE[@]}" -eq "${#INACTIVE[@]}" ]; then + STATUS="SKIPPED" + REASON="The required plugin ${STILL_INACTIVE[0]} could not be activated" + break + fi + + INACTIVE=( ${STILL_INACTIVE[@]+"${STILL_INACTIVE[@]}"} ) + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so the + # baseline is checked before it is activated. A site that is already broken says something about the + # dependency rather than about the plugin being tested, and blaming the plugin for it would be the + # masking these checks are built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # A dependency can fatal after the page has started to render, which still answers with a 200. + # The fatal only shows up in the log, and it would show up again once the plugin under test is + # active and be blamed on that plugin, so the log is checked before it is cleared. + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" + fi + + # The plugin under test is activated through WP-CLI, which loads the dependencies first. One that is + # healthy over HTTP but cannot be loaded by WP-CLI, like the plugins described in step 8, would + # stop that activation and have its failure blamed on the plugin under test. + if [ "${STATUS}" = "PASS" ]; then + DEPENDENCY_EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${DEPENDENCY_EVAL_OUTPUT#*loaded-ok}" = "${DEPENDENCY_EVAL_OUTPUT}" ]; then + printf '%s\n' "${DEPENDENCY_EVAL_OUTPUT}" + STATUS="SKIPPED" + REASON="WP-CLI cannot load WordPress with the required plugins active, so the plugin cannot be activated" + fi + fi + + # Anything else the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. + rm -f "${WP_DIR}/wp-content/debug.log" + fi + + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. + # Dependencies are installed above, so what is left here is a plugin asking for a + # version of PHP or WordPress this run is not using, or for a dependency that is not on + # WordPress.org. That is core working as designed rather than a fatal, so it is recorded + # as skipped. Core says "does not meet" when one of the two is unmet and "do not meet" when + # both are. + *"to be installed and activated"* | *"does not meet minimum requirements"* | *"do not meet minimum requirements"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + elif ! is_active "${SLUG}"; then + # Checking the site now would be checking it without the plugin, and passing it for that. + STATUS="SKIPPED" + REASON="WP-CLI reported success, but the plugin was not active afterwards, so its activation stopped early" + fi + fi + + # Step 6: request the front page and the login screen through the PHP built-in server. A real request + # is what decides whether a plugin passed, because it is what a visitor gets. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 7: a fatal or parse error can be logged without changing the HTTP status, for example during + # a shutdown hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" + STATUS="FAIL" + REASON="A fatal error was logged: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" + fi + + # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a + # plugin on its own. + # + # WP-CLI requires `wp-settings.php` from inside a method, so a plugin that assigns a variable at file + # scope and reads it back with `global` later finds nothing there. Plenty of plugins do exactly that, + # and the resulting fatal happens only under WP-CLI - a visitor never sees it. Failing a plugin for + # it would report breakage that does not exist on a real site, so it is recorded as a note against a + # plugin that is otherwise healthy. Anything that genuinely fatals on load fails the HTTP checks + # above. + # + # The fatal is written to the debug log too, which is why this comes after the log has been checked. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"Parse error"* | *"Uncaught"* ) + REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + reset_site || reset_failed "${SLUG}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + group_end + done <> "${GITHUB_STEP_SUMMARY}" + fi + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf '\nThe following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" + return 1 + fi + + printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" +} + +## +# Compares each plugin's result with the one the expected results file lists for it, and exits non zero when +# any of them differ. +# +# Each line of the file is a slug, the expected result, and optionally some text that the "Also active" or +# "Details" column has to contain. That last part is what tells a plugin that failed for the reason it was +# built to fail apart from one that failed for some other reason. +## +check_expectations() { + MISMATCH_COUNT=0 + MATCH_COUNT=0 + + printf '\nChecking the results against %s.\n\n' "${EXPECT_FILE}" + + while read -r EXPECTED_SLUG EXPECTED_STATUS EXPECTED_TEXT; do + case "${EXPECTED_SLUG}" in + '' | '#'* ) continue ;; + esac + + ROW="$( awk -F '\t' -v slug="${EXPECTED_SLUG}" '$1 == slug { print; exit }' "${RESULTS}" )" + ACTUAL_STATUS="$( printf '%s' "${ROW}" | cut -f 3 )" + ACTUAL_TEXT="$( printf '%s' "${ROW}" | cut -f 4- | tr '\t' ' ' )" + MATCHED="yes" + + if [ -z "${ROW}" ] || [ "${ACTUAL_STATUS}" != "${EXPECTED_STATUS}" ]; then + MATCHED="no" + elif [ -n "${EXPECTED_TEXT}" ]; then + case "${ACTUAL_TEXT}" in + *"${EXPECTED_TEXT}"* ) ;; + * ) MATCHED="no" ;; + esac + fi + + if [ "${MATCHED}" = "yes" ]; then + MATCH_COUNT=$(( MATCH_COUNT + 1 )) + printf 'ok %s: %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" + else + MISMATCH_COUNT=$(( MISMATCH_COUNT + 1 )) + printf 'MISMATCH %s: expected %s %s\n got %s %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" "${EXPECTED_TEXT}" "${ACTUAL_STATUS:-no result}" "${ACTUAL_TEXT}" + fi + done < "${EXPECT_FILE}" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s of %s results matched %s.\n\n' "${MATCH_COUNT}" "$(( MATCH_COUNT + MISMATCH_COUNT ))" "$( basename "${EXPECT_FILE}" )" >> "${GITHUB_STEP_SUMMARY}" + fi + + if [ "${MISMATCH_COUNT}" -gt 0 ]; then + printf '\n%s result(s) did not match what was expected.\n' "${MISMATCH_COUNT}" + return 1 + fi + + printf '\nEvery result matched what was expected.\n' +} + +if [ "${USE_DOCKER}" = "yes" ]; then + run_in_docker + exit $? +fi + +run_checks