From 88d6003f0af4dc3182e511b3f8abd99867a7a844 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 30 Oct 2026 20:54:42 +0100 Subject: [PATCH 1/5] Run parallel workers in throwaway Docker containers With WP_CLI_TEST_DOCKER set and Docker available, every worker runs in its own container, next to its own database server container of the same type and version as the configured one. Workers share no network, temporary directory or database, so any of them can run any scenario, including the ones using fixed ports or fixed temporary paths. The database servers keep their data in memory. The containers run the same PHP version as the host, which the project's Composer dependencies are installed for, and use WordPress as downloaded once by the host. Where the image cannot be built or the database server cannot be identified, the scenarios run without Docker. Co-Authored-By: Claude Opus 5.5 --- .readme-partials/USING.md | 7 ++ bin/run-behat-tests | 18 ++- utils/behat-parallel.php | 258 +++++++++++++++++++++++++++++++++++++- utils/docker/Dockerfile | 15 +++ 4 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 utils/docker/Dockerfile diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 80d8127f..f5592b58 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -265,6 +265,13 @@ Every worker but the first gets its own temporary directory, and with it its own Scenarios that rely on resources shared across the system, such as a fixed port or a fixed path in the system's temporary directory, are all run by the first worker, one after the other. `composer behat-rerun` re-runs exactly the scenarios that failed in the last parallel run. +When Docker is available, set `WP_CLI_TEST_DOCKER` to run every worker in its own throwaway containers instead: +```bash +WP_CLI_TEST_DOCKER=1 WP_CLI_TEST_PARALLEL=4 composer behat +``` + +Each worker then gets its own network, temporary directory and database server, of the same type and version as the one configured through the `WP_CLI_TEST_DB*` variables (or set the image through `WP_CLI_TEST_DOCKER_DB`, e.g. `mysql:8.0`). Nothing is shared between workers, so they can run any scenario. The database servers keep their data in memory. The containers run the same PHP version as the host, as the project's Composer dependencies are installed for it. Where the image cannot be built or the database server cannot be identified, the scenarios run without Docker. + ### Linting the feature files `composer lint-gherkin` checks `features/` with diff --git a/bin/run-behat-tests b/bin/run-behat-tests index b1600b38..f8053233 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -286,8 +286,24 @@ done case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) - PARALLEL="";; + PARALLEL="" + unset WP_CLI_TEST_DOCKER;; esac + +# With WP_CLI_TEST_DOCKER set, every worker runs in its own throwaway containers instead, +# with a database server of the same type and version, so they share nothing. +if [ -n "${WP_CLI_TEST_DOCKER-}" ]; then + if ! docker info > /dev/null 2>&1; then + echo "Warning: WP_CLI_TEST_DOCKER is set, but Docker is not available. Running the scenarios without it." >&2 + unset WP_CLI_TEST_DOCKER + elif [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]]; then + echo "Code coverage is collected, running the scenarios without Docker." >&2 + unset WP_CLI_TEST_DOCKER + else + exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL:-1}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" + fi +fi + if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then if ! is_numeric "${PARALLEL}"; then echo "Warning: WP_CLI_TEST_PARALLEL is not a number of workers, running the scenarios one after the other." >&2 diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index 86c13c8c..bdfd674e 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -187,6 +187,179 @@ static function ( $a, $b ) { return $shards; } +/** + * Get the Docker image of the database server the tests run against on the host, so that the + * containers test against the same type and version of database server. + * + * @return string|null Image name, or null if the server could not be identified. + */ +function wp_cli_tests_docker_db_image() { + $image = getenv( 'WP_CLI_TEST_DOCKER_DB' ); + if ( is_string( $image ) && '' !== $image ) { + return $image; + } + + $host = getenv( 'WP_CLI_TEST_DBHOST' ) ?: 'localhost'; + $parts = explode( ':', $host ); + $args = '-h' . escapeshellarg( $parts[0] ); + if ( isset( $parts[1] ) ) { + $args .= is_numeric( $parts[1] ) ? ' --port=' . escapeshellarg( $parts[1] ) . ' --protocol=tcp' : ' --socket=' . escapeshellarg( $parts[1] ); + } + $pass = getenv( 'WP_CLI_TEST_DBROOTPASS' ); + $args .= ' -u' . escapeshellarg( getenv( 'WP_CLI_TEST_DBROOTUSER' ) ?: 'root' ) . ( false !== $pass && '' !== $pass ? ' -p' . escapeshellarg( $pass ) : '' ); + + foreach ( [ 'mysql', 'mariadb' ] as $client ) { + $version = (string) exec( "{$client} {$args} --skip-column-names -e 'SELECT VERSION()' 2>/dev/null" ); + if ( preg_match( '/^(\d+\.\d+)/', $version, $matches ) ) { + return ( false !== stripos( $version, 'mariadb' ) ? 'mariadb:' : 'mysql:' ) . $matches[1]; + } + } + + return null; +} + +/** + * Get the extra options a database server image needs to accept the test user's password. + * + * @param string $image + * @return string + */ +function wp_cli_tests_docker_db_options( $image ) { + // Throwaway databases do not need to wait for their writes to reach the disk. + $options = '--innodb-flush-log-at-trx-commit=0 --sync-binlog=0'; + + if ( preg_match( '/^mysql:(\d+)\.(\d+)/', $image, $matches ) && (int) $matches[1] >= 8 ) { + $options .= version_compare( "{$matches[1]}.{$matches[2]}", '8.4', '>=' ) + ? ' --mysql-native-password=ON --authentication-policy=mysql_native_password' + : ' --default-authentication-plugin=mysql_native_password'; + } + + return $options; +} + +/** + * Build the image the workers run in. + * + * @return string|null Image name, or null if it could not be built. + */ +function wp_cli_tests_docker_php_image() { + $version = PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION; + $image = "wp-cli-tests-php:{$version}"; + + exec( 'docker build -q -t ' . escapeshellarg( $image ) . ' --build-arg ' . escapeshellarg( "PHP_VERSION={$version}" ) . ' ' . escapeshellarg( __DIR__ . '/docker' ) . ' 2>&1', $output, $exit_code ); + + if ( 0 !== $exit_code ) { + fwrite( STDERR, "Warning: Could not build the Docker image for PHP {$version}:\n" . implode( "\n", array_slice( $output, -5 ) ) . "\n" ); + return null; + } + + return $image; +} + +/** + * Get the directories a container needs to see at their usual paths. + * + * @param string $base_dir + * @return array Paths to mount at the same path, and read-only sources keyed by source path. + */ +function wp_cli_tests_docker_mounts( $base_dir ) { + $cwd = (string) getcwd(); + $mounts = [ $cwd, $base_dir ]; + + // Dependencies can be symlinked from outside the project, as in wp-cli/wp-cli-dev, + // in which case the closest directory holding both has to be visible. + foreach ( [ $cwd . '/vendor/autoload.php' ] as $path ) { + $real = realpath( $path ); + if ( false === $real || 0 === strpos( $real . '/', $cwd . '/' ) ) { + continue; + } + $common = $cwd; + while ( 0 !== strpos( $real . '/', rtrim( $common, '/' ) . '/' ) && dirname( $common ) !== $common ) { + $common = dirname( $common ); + } + $mounts[] = '/' === $common ? dirname( $real ) : $common; + } + + $mounts = array_values( array_unique( $mounts ) ); + + // Download WordPress once for all containers, which only ever copy from it. + $version = getenv( 'WP_VERSION' ); + if ( $version && ! getenv( 'WP_CLI_TEST_CORE_ZIP' ) ) { + $cache = sys_get_temp_dir() . '/wp-cli-test-core-download-cache-' . $version; + if ( ! is_readable( $cache . '/wp-includes/version.php' ) ) { + exec( 'vendor/bin/wp core download --force --path=' . escapeshellarg( $cache ) . ' --version=' . escapeshellarg( $version ) . ' > /dev/null 2>&1' ); + } + if ( is_readable( $cache . '/wp-includes/version.php' ) ) { + $mounts[ $cache ] = '/tmp/wp-cli-test-core-download-cache-' . $version; + } + } + + return $mounts; +} + +/** + * Start the containers of a worker and get the command that runs its scenarios. + * + * @param string $name Name prefix of the worker's containers. + * @param string $php_image + * @param string|null $db_image Null when running on SQLite. + * @param array $mounts + * @param array $user_args + * @param string $scenario_list File listing the worker's scenarios. + * @return string + */ +function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array $mounts, array $user_args, $scenario_list ) { + $network = ''; + + if ( null !== $db_image ) { + exec( + 'docker run -d --rm --name ' . escapeshellarg( "{$name}-db" ) + . ' --tmpfs /var/lib/mysql' + . ' -e MYSQL_ROOT_PASSWORD=root -e MYSQL_USER=wp_cli_test -e MYSQL_PASSWORD=password1' + . ' ' . escapeshellarg( $db_image ) . ' ' . wp_cli_tests_docker_db_options( $db_image ) . ' 2>&1' + ); + // Share the database container's network, so the server is at 127.0.0.1 and nothing else is. + $network = ' --network ' . escapeshellarg( "container:{$name}-db" ); + } + + $env = [ + 'HOME' => '/tmp/home', + 'WP_CLI_TEST_DBHOST' => '127.0.0.1:3306', + 'WP_CLI_TEST_DBROOTUSER' => 'root', + 'WP_CLI_TEST_DBROOTPASS' => 'root', + ]; + + foreach ( getenv() as $key => $value ) { + if ( preg_match( '/^(WP_VERSION|NO_COLOR|CI|GITHUB_TOKEN|WP_CLI_TEST_(DBTYPE|OBJECT_CACHE|CORE_ZIP|QUIET|LOG_RUN_TIMES|PHP_TUNING|WP_VERSION_CACHE_TTL))$/', $key ) ) { + $env[ $key ] = $value; + } + } + + $command = 'docker run --rm --name ' . escapeshellarg( $name ) . $network + . ' --tmpfs /tmp:exec,mode=1777' + . ' -u ' . escapeshellarg( getmyuid() . ':' . getmygid() ) + . ' -w ' . escapeshellarg( (string) getcwd() ); + + foreach ( $mounts as $source => $target ) { + if ( is_int( $source ) ) { + $command .= ' -v ' . escapeshellarg( "{$target}:{$target}" ); + } else { + $command .= ' -v ' . escapeshellarg( "{$source}:{$target}:ro" ); + } + } + + foreach ( $env as $key => $value ) { + $command .= ' -e ' . escapeshellarg( "{$key}={$value}" ); + } + + $inner = 'mkdir -p "$HOME"' + // Wait for the database server, which takes a few seconds to start, before preparing it. + . ( null !== $db_image ? ' && for i in $(seq 1 60); do mysql -h127.0.0.1 -P3306 --protocol=tcp -uroot -proot -e "SELECT 1" > /dev/null 2>&1 && break; sleep 1; done && vendor/bin/install-package-tests > /dev/null' : '' ) + . ' && vendor/bin/run-behat-tests ' . implode( ' ', array_map( 'escapeshellarg', array_merge( $user_args, [ $scenario_list ] ) ) ); + + return $command . ' ' . escapeshellarg( $php_image ) . ' sh -c ' . escapeshellarg( $inner ); +} + $args = array_slice( $argv, 1 ); $workers = (int) array_shift( $args ); @@ -196,9 +369,18 @@ static function ( $a, $b ) { $user_args = false === $separator ? [] : array_slice( $args, $separator + 1 ); $all_args = array_merge( $behat_args, $user_args ); +// Run every worker in its own throwaway containers, see run-behat-tests. +$docker = (bool) getenv( 'WP_CLI_TEST_DOCKER' ) && DIRECTORY_SEPARATOR !== '\\'; +$rerun_paths = null; + // A rerun after a parallel run covers exactly the scenarios that failed in it. $rerun = array_search( '--rerun', $user_args, true ); -if ( false !== $rerun && file_exists( wp_cli_tests_failures_file() ) ) { +if ( false !== $rerun && file_exists( wp_cli_tests_failures_file() ) && $docker ) { + unset( $user_args[ $rerun ] ); + $user_args = array_values( $user_args ); + $rerun_paths = array_filter( explode( "\n", (string) file_get_contents( wp_cli_tests_failures_file() ) ) ); + unlink( wp_cli_tests_failures_file() ); +} elseif ( false !== $rerun && file_exists( wp_cli_tests_failures_file() ) ) { unset( $user_args[ $rerun ] ); $failures = wp_cli_tests_failures_file(); $copy = $failures . '.' . getmypid() . '.scenarios'; @@ -217,10 +399,25 @@ static function ( $a, $b ) { } } -if ( $workers < 2 || DIRECTORY_SEPARATOR === '\\' || $selects_scenarios ) { +if ( ( $workers < 2 && ! $docker ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths ) ) { wp_cli_tests_run_behat_and_exit( $all_args ); } +$php_image = null; +$db_image = null; +if ( $docker ) { + $php_image = wp_cli_tests_docker_php_image(); + $db_image = 'sqlite' === getenv( 'WP_CLI_TEST_DBTYPE' ) ? null : wp_cli_tests_docker_db_image(); + + if ( null === $php_image || ( null === $db_image && 'sqlite' !== getenv( 'WP_CLI_TEST_DBTYPE' ) ) ) { + fwrite( STDERR, "Warning: Running the scenarios without Docker.\n" ); + $docker = false; + if ( $workers < 2 || null !== $rerun_paths ) { + wp_cli_tests_run_behat_and_exit( $all_args ); + } + } +} + require getcwd() . '/vendor/autoload.php'; $tags = ''; @@ -230,14 +427,43 @@ static function ( $a, $b ) { } } +// Containers check for PHP extensions themselves, as theirs can differ from the host's. +if ( $docker ) { + $tags = (string) preg_replace( '/(^|&&)~@require-extension-[A-Za-z0-9_]+/', '', $tags ); + $tags = trim( $tags, '&' ); +} + $features_folder = getenv( 'BEHAT_FEATURES_FOLDER' ) ?: 'features'; -$scenarios = wp_cli_tests_collect_scenarios( $features_folder, $tags ); -$workers = min( $workers, max( 1, count( $scenarios ) ) ); -if ( $workers < 2 ) { +if ( null !== $rerun_paths ) { + $scenarios = array_map( + static function ( $path ) { + return [ + 'path' => $path, + 'cost' => 1, + 'shared' => false, + ]; + }, + $rerun_paths + ); + $workers = 1; +} else { + $scenarios = wp_cli_tests_collect_scenarios( $features_folder, $tags ); +} + +$workers = min( max( 1, $workers ), max( 1, count( $scenarios ) ) ); + +if ( $workers < 2 && ! $docker ) { wp_cli_tests_run_behat_and_exit( $all_args ); } +// Containers do not share ports or temporary directories, so any of them can run any scenario. +if ( $docker ) { + foreach ( $scenarios as $key => $scenario ) { + $scenarios[ $key ]['shared'] = false; + } +} + $shards = wp_cli_tests_distribute( $scenarios, $workers ); // Kept between runs, so that the workers' caches stay warm and failed scenarios can be inspected. $base_dir = sys_get_temp_dir() . '/wp-cli-test-parallel-' . substr( md5( (string) getcwd() ), 0, 8 ); @@ -247,7 +473,19 @@ static function ( $a, $b ) { mkdir( $base_dir, 0777, true ); } -printf( "Running %d scenarios in %d parallel workers.\n", count( $scenarios ), $workers ); +printf( "Running %d scenarios in %d parallel workers%s.\n", count( $scenarios ), $workers, $docker ? ' in Docker containers' : '' ); + +$containers = []; +if ( $docker ) { + $mounts = wp_cli_tests_docker_mounts( $base_dir ); + register_shutdown_function( + static function () use ( &$containers ) { + foreach ( $containers as $container ) { + exec( 'docker rm -f ' . escapeshellarg( $container ) . ' > /dev/null 2>&1' ); + } + } + ); +} $procs = []; foreach ( $shards as $index => $paths ) { @@ -267,6 +505,14 @@ static function ( $a, $b ) { $output = "{$base_dir}/worker-{$index}.log"; $command = escapeshellarg( PHP_BINARY ) . ' vendor/bin/behat ' . implode( ' ', array_map( 'escapeshellarg', array_merge( $all_args, [ $list ] ) ) ); + if ( $docker ) { + $name = 'wp-cli-tests-' . substr( md5( $base_dir ), 0, 8 ) . "-{$index}"; + $containers[] = $name; + $containers[] = "{$name}-db"; + $env = getenv(); + $command = wp_cli_tests_docker_worker_command( $name, (string) $php_image, $db_image, $mounts, array_values( $user_args ), $list ); + } + $procs[ $index ] = [ 'proc' => proc_open( $command, diff --git a/utils/docker/Dockerfile b/utils/docker/Dockerfile new file mode 100644 index 00000000..bcc4ca45 --- /dev/null +++ b/utils/docker/Dockerfile @@ -0,0 +1,15 @@ +# Image for running the Behat scenarios of a WP-CLI package in throwaway containers. +# Its PHP version has to match the one the package's Composer dependencies were installed for. +ARG PHP_VERSION=8.3 +FROM php:${PHP_VERSION}-cli + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl git jq less mariadb-client subversion unzip vim-tiny zip \ + libfreetype6-dev libjpeg-dev libpng-dev libzip-dev \ + && docker-php-ext-configure gd --with-freetype --with-jpeg \ + && docker-php-ext-install exif gd mysqli pdo_mysql zip \ + && ( php -m | grep -qi '^zend opcache$' || docker-php-ext-install opcache ) \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=composer:2 /usr/bin/composer /usr/bin/composer From cab964ef2a32f191c36f49e102320dde0cee91a3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 31 Oct 2026 11:12:00 +0100 Subject: [PATCH 2/5] Give the containers the host's PHP error settings Test outcomes depend on how PHP reports errors. Without a php.ini, the containers reported E_ALL, which WP-CLI keeps while testing, whereas the CI runners report -1, which WP-CLI reduces to everything but deprecations. Scenarios running older WordPress versions on PHP 8 therefore failed in the containers on deprecation notices that the runners never print. Co-Authored-By: Claude Opus 5.5 --- utils/behat-parallel.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index bdfd674e..a0c018c4 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -294,6 +294,21 @@ function wp_cli_tests_docker_mounts( $base_dir ) { } } + // Give the containers the PHP settings of the host for how errors are reported, as test + // outcomes depend on them. WP-CLI, for example, only reports deprecations while testing + // when error_reporting is exactly E_ALL, which a missing php.ini leads to, but not -1. + $settings = []; + foreach ( [ 'error_reporting', 'display_errors', 'display_startup_errors', 'log_errors', 'html_errors', 'zend.assertions', 'memory_limit', 'date.timezone' ] as $name ) { + $value = ini_get( $name ); + // An empty value is a setting that is switched off, which has to be carried over as well. + if ( false !== $value ) { + $settings[] = $name . ' = ' . ( is_numeric( $value ) ? $value : '"' . addcslashes( $value, '"\\' ) . '"' ); + } + } + $ini = "{$base_dir}/php-host.ini"; + file_put_contents( $ini, implode( "\n", $settings ) . "\n" ); + $mounts[ $ini ] = '/usr/local/etc/php/conf.d/zz-wp-cli-tests-host.ini'; + return $mounts; } From f7e9c1df27ab1d86aa521ff07e55a251498ce951 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 31 Oct 2026 18:29:11 +0100 Subject: [PATCH 3/5] Install ps in the image, and the packages of archived Debian releases Background processes such as the built-in web server are stopped through ps, without which they keep their port for the next scenario. Co-Authored-By: Claude Opus 5.5 --- utils/docker/Dockerfile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/utils/docker/Dockerfile b/utils/docker/Dockerfile index bcc4ca45..137d0b8a 100644 --- a/utils/docker/Dockerfile +++ b/utils/docker/Dockerfile @@ -3,10 +3,17 @@ ARG PHP_VERSION=8.3 FROM php:${PHP_VERSION}-cli -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - curl git jq less mariadb-client subversion unzip vim-tiny zip \ - libfreetype6-dev libjpeg-dev libpng-dev libzip-dev \ +# Images of older PHP versions are based on Debian releases whose packages have moved to the archive. +RUN install_packages() { \ + apt-get update && apt-get install -y --no-install-recommends \ + curl git jq less mariadb-client procps subversion unzip vim-tiny zip \ + libfreetype6-dev libjpeg-dev libpng-dev libzip-dev; \ + } \ + && ( install_packages || { \ + sed -i -e 's|deb.debian.org|archive.debian.org|g' -e '/-updates/d' /etc/apt/sources.list \ + && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive \ + && install_packages; \ + } ) \ && docker-php-ext-configure gd --with-freetype --with-jpeg \ && docker-php-ext-install exif gd mysqli pdo_mysql zip \ && ( php -m | grep -qi '^zend opcache$' || docker-php-ext-install opcache ) \ From bb266f4a02c54e42aec92295989a2ab557682203 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 31 Oct 2026 18:29:11 +0100 Subject: [PATCH 4/5] Let Docker be required, used where possible, or not used, and run any PHP version WP_CLI_TEST_DOCKER=auto runs the scenarios without Docker where it cannot be used, and reports why. 1 fails instead. Scenarios selected by path or name run in a single container. WP_CLI_TEST_DOCKER_PHP runs the containers on another PHP version than the host's, with the Composer dependencies installed for it. Options taking a value, as in --format progress, are no longer mistaken for paths. Co-Authored-By: Claude Opus 5.5 --- .readme-partials/USING.md | 21 +++- bin/run-behat-tests | 57 +++++++---- utils/behat-parallel.php | 208 +++++++++++++++++++++++++++++++++----- 3 files changed, 243 insertions(+), 43 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index f5592b58..f6201a47 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -265,12 +265,27 @@ Every worker but the first gets its own temporary directory, and with it its own Scenarios that rely on resources shared across the system, such as a fixed port or a fixed path in the system's temporary directory, are all run by the first worker, one after the other. `composer behat-rerun` re-runs exactly the scenarios that failed in the last parallel run. -When Docker is available, set `WP_CLI_TEST_DOCKER` to run every worker in its own throwaway containers instead: +Set `WP_CLI_TEST_DOCKER` to run the scenarios in throwaway Docker containers instead, every worker in its own: ```bash -WP_CLI_TEST_DOCKER=1 WP_CLI_TEST_PARALLEL=4 composer behat +WP_CLI_TEST_DOCKER=auto WP_CLI_TEST_PARALLEL=4 composer behat ``` -Each worker then gets its own network, temporary directory and database server, of the same type and version as the one configured through the `WP_CLI_TEST_DB*` variables (or set the image through `WP_CLI_TEST_DOCKER_DB`, e.g. `mysql:8.0`). Nothing is shared between workers, so they can run any scenario. The database servers keep their data in memory. The containers run the same PHP version as the host, as the project's Composer dependencies are installed for it. Where the image cannot be built or the database server cannot be identified, the scenarios run without Docker. +| `WP_CLI_TEST_DOCKER` | Effect | +|---------------------------|--------------------------------------------------------------------------------------------------| +| not set, `0`, `no`, `off` | The scenarios run without Docker. | +| `auto` | The scenarios run in containers where possible, and without Docker otherwise, which is reported. | +| `1`, `yes`, `on` | The scenarios run in containers, and the run fails where that is not possible. | + +Docker is not used on Windows, when collecting code coverage, when Docker is not available, when the image cannot be built or when the database server cannot be identified. + +Each worker gets its own network, temporary directory and database server, of the same type and version as the one configured through the `WP_CLI_TEST_DB*` variables (or set the image through `WP_CLI_TEST_DOCKER_DB`, e.g. `mysql:8.0`). Nothing is shared between workers, so they can run any scenario. The database servers keep their data in memory. Scenarios selected by path or name run in a single container. + +The containers run the same PHP version as the host, unless `WP_CLI_TEST_DOCKER_PHP` sets another one: +```bash +WP_CLI_TEST_DOCKER=1 WP_CLI_TEST_DOCKER_PHP=7.4 WP_CLI_TEST_PARALLEL=4 composer behat +``` + +The project's Composer dependencies are then installed for that PHP version in a separate directory, the way CI installs them without a lock file, and refreshed once a day. Dependencies that are linked to local checkouts, as in `wp-cli/wp-cli-dev`, come from their published versions in that case, which is reported. ### Linting the feature files diff --git a/bin/run-behat-tests b/bin/run-behat-tests index f8053233..c86c8214 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -276,34 +276,57 @@ done # to be allowed to create, as set up by "composer prepare-tests". PARALLEL="${WP_CLI_TEST_PARALLEL-}" -# A rerun after a parallel run covers exactly the scenarios that failed in it, which only the -# parallel runner knows, whether or not WP_CLI_TEST_PARALLEL is set for the rerun as well. -for arg in "$@"; do - if [ "$arg" = "--rerun" ] && [ -f "${TMPDIR:-/tmp}/wp-cli-test-parallel-failures.scenarios" ]; then - exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php 1 --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" +# With WP_CLI_TEST_DOCKER set, the scenarios run in throwaway containers instead, every worker in +# its own, with a database server of the same type and version, so they share nothing. +# "auto" runs them without Docker where it cannot be used, "1" (or "yes", "true", "on") fails instead. +case "${WP_CLI_TEST_DOCKER-}" in + ""|0|no|false|off) DOCKER="";; + auto) DOCKER="auto";; + 1|yes|true|on) DOCKER="force";; + *) + echo "Warning: WP_CLI_TEST_DOCKER is neither \"auto\" nor a yes or no, running the scenarios without Docker." >&2 + DOCKER="";; +esac + +without_docker() { + if [ "${DOCKER}" = "force" ]; then + echo "Error: $1 Set WP_CLI_TEST_DOCKER to \"auto\" to run the scenarios without Docker in that case." >&2 + exit 1 fi -done + echo "$1 Running the scenarios without Docker." >&2 + DOCKER="" +} case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) PARALLEL="" - unset WP_CLI_TEST_DOCKER;; + if [ -n "${DOCKER}" ]; then + without_docker "The scenarios cannot run in Docker containers on Windows." + fi;; esac -# With WP_CLI_TEST_DOCKER set, every worker runs in its own throwaway containers instead, -# with a database server of the same type and version, so they share nothing. -if [ -n "${WP_CLI_TEST_DOCKER-}" ]; then - if ! docker info > /dev/null 2>&1; then - echo "Warning: WP_CLI_TEST_DOCKER is set, but Docker is not available. Running the scenarios without it." >&2 - unset WP_CLI_TEST_DOCKER +if [ -n "${DOCKER}" ]; then + if ! command -v docker > /dev/null 2>&1 || ! docker info > /dev/null 2>&1; then + without_docker "Docker is not available." elif [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]]; then - echo "Code coverage is collected, running the scenarios without Docker." >&2 - unset WP_CLI_TEST_DOCKER - else - exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL:-1}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" + without_docker "Code coverage cannot be collected in Docker containers." fi fi +export WP_CLI_TEST_DOCKER="${DOCKER}" + +# A rerun after a parallel run covers exactly the scenarios that failed in it, which only the +# parallel runner knows, whether or not WP_CLI_TEST_PARALLEL is set for the rerun as well. +for arg in "$@"; do + if [ "$arg" = "--rerun" ] && [ -f "${TMPDIR:-/tmp}/wp-cli-test-parallel-failures.scenarios" ]; then + exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php 1 --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" + fi +done + +if [ -n "${DOCKER}" ]; then + exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL:-1}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" +fi + if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then if ! is_numeric "${PARALLEL}"; then echo "Warning: WP_CLI_TEST_PARALLEL is not a number of workers, running the scenarios one after the other." >&2 diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index a0c018c4..fec809e1 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -41,6 +41,36 @@ function wp_cli_tests_failures_file() { return sys_get_temp_dir() . '/wp-cli-test-parallel-failures.scenarios'; } +/** + * Get whether the scenarios run in Docker containers, as set through WP_CLI_TEST_DOCKER. + * + * @return string "auto" to run them without Docker where it cannot be used, "force" to fail + * instead, or an empty string to run them without Docker. + */ +function wp_cli_tests_docker_mode() { + $value = strtolower( (string) getenv( 'WP_CLI_TEST_DOCKER' ) ); + + if ( in_array( $value, [ 'auto', 'force' ], true ) ) { + return $value; + } + + return in_array( $value, [ '1', 'yes', 'true', 'on' ], true ) ? 'force' : ''; +} + +/** + * Report that Docker cannot be used, and fail if it is required. + * + * @param string $reason + */ +function wp_cli_tests_without_docker( $reason ) { + if ( 'force' === wp_cli_tests_docker_mode() ) { + fwrite( STDERR, "Error: {$reason} Set WP_CLI_TEST_DOCKER to \"auto\" to run the scenarios without Docker in that case.\n" ); + exit( 1 ); + } + + fwrite( STDERR, "{$reason} Running the scenarios without Docker.\n" ); +} + /** * Run Behat in the current process' environment and exit with its exit code. * @@ -237,14 +267,35 @@ function wp_cli_tests_docker_db_options( $image ) { return $options; } +/** + * Get the PHP version the workers run, which is the host's unless WP_CLI_TEST_DOCKER_PHP sets another one. + * + * @return string Major and minor version. + */ +function wp_cli_tests_docker_php_version() { + $host = PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION; + $version = getenv( 'WP_CLI_TEST_DOCKER_PHP' ); + + if ( ! is_string( $version ) || '' === $version ) { + return $host; + } + + if ( ! preg_match( '/^\d+\.\d+$/', $version ) ) { + fwrite( STDERR, "Warning: WP_CLI_TEST_DOCKER_PHP is not a PHP version like 7.4, running PHP {$host} in the containers.\n" ); + return $host; + } + + return $version; +} + /** * Build the image the workers run in. * + * @param string $version PHP version. * @return string|null Image name, or null if it could not be built. */ -function wp_cli_tests_docker_php_image() { - $version = PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION; - $image = "wp-cli-tests-php:{$version}"; +function wp_cli_tests_docker_php_image( $version ) { + $image = "wp-cli-tests-php:{$version}"; exec( 'docker build -q -t ' . escapeshellarg( $image ) . ' --build-arg ' . escapeshellarg( "PHP_VERSION={$version}" ) . ' ' . escapeshellarg( __DIR__ . '/docker' ) . ' 2>&1', $output, $exit_code ); @@ -256,19 +307,81 @@ function wp_cli_tests_docker_php_image() { return $image; } +/** + * Install the project's Composer dependencies for another PHP version than the host's, as the + * versions installed for the host can require a newer PHP version or not support an older one. + * + * The dependencies are resolved the way CI does without a lock file, and refreshed once a day. + * + * @param string $php_image Image of the PHP version to install them for. + * @param string $version PHP version. + * @return string|null Vendor directory, or null if the dependencies could not be installed. + */ +function wp_cli_tests_docker_vendor( $php_image, $version ) { + $cwd = (string) getcwd(); + $json = (string) file_get_contents( $cwd . '/composer.json' ); + $dir = sys_get_temp_dir() . "/wp-cli-test-docker-vendor-{$version}-" . substr( md5( $cwd . "\n" . $json ), 0, 12 ); + + // Installed dependencies can be linked to local checkouts, as in wp-cli/wp-cli-dev, + // which a separate installation cannot use. + $linked = glob( $cwd . '/vendor/*/*', GLOB_ONLYDIR ) ?: []; + $linked = array_filter( $linked, 'is_link' ); + if ( $linked ) { + fwrite( STDERR, "Warning: The containers running PHP {$version} use the published versions of the dependencies instead of the local checkouts they are linked to, for example " . substr( reset( $linked ), strlen( $cwd ) + 1 ) . ".\n" ); + } + + $marker = "{$dir}/installed"; + if ( file_exists( $marker ) && filemtime( $marker ) > time() - 86400 ) { + return "{$dir}/vendor"; + } + + if ( ! is_dir( "{$dir}/vendor" ) ) { + mkdir( "{$dir}/vendor", 0777, true ); + } + file_put_contents( "{$dir}/composer.json", $json ); + + $cache = getenv( 'COMPOSER_CACHE_DIR' ) ?: ( getenv( 'HOME' ) . '/.cache/composer' ); + + fwrite( STDERR, "Installing the Composer dependencies for PHP {$version}...\n" ); + + // The project is read-only, and Composer writes its lock file next to the copy of composer.json. + $command = 'docker run --rm' + . ' -u ' . escapeshellarg( getmyuid() . ':' . getmygid() ) + . ' -v ' . escapeshellarg( "{$cwd}:{$cwd}:ro" ) + . ' -v ' . escapeshellarg( "{$dir}:/wp-cli-tests-vendor" ) + . ' -v ' . escapeshellarg( "{$dir}/vendor:{$cwd}/vendor" ) + . ( is_dir( $cache ) ? ' -v ' . escapeshellarg( "{$cache}:/tmp/composer-cache" ) . ' -e COMPOSER_CACHE_DIR=/tmp/composer-cache' : '' ) + . ' -e HOME=/tmp -e COMPOSER=/wp-cli-tests-vendor/composer.json' + . ' -w ' . escapeshellarg( $cwd ) + . ' ' . escapeshellarg( $php_image ) + . ' composer update --no-interaction --no-progress 2>&1'; + + exec( $command, $output, $exit_code ); + + if ( 0 !== $exit_code ) { + fwrite( STDERR, "Warning: Could not install the Composer dependencies for PHP {$version}:\n" . implode( "\n", array_slice( $output, -5 ) ) . "\n" ); + return null; + } + + touch( $marker ); + + return "{$dir}/vendor"; +} + /** * Get the directories a container needs to see at their usual paths. * - * @param string $base_dir + * @param string $base_dir + * @param string|null $vendor Dependencies installed for the PHP version of the containers, if not the host's. * @return array Paths to mount at the same path, and read-only sources keyed by source path. */ -function wp_cli_tests_docker_mounts( $base_dir ) { +function wp_cli_tests_docker_mounts( $base_dir, $vendor = null ) { $cwd = (string) getcwd(); $mounts = [ $cwd, $base_dir ]; // Dependencies can be symlinked from outside the project, as in wp-cli/wp-cli-dev, // in which case the closest directory holding both has to be visible. - foreach ( [ $cwd . '/vendor/autoload.php' ] as $path ) { + foreach ( null === $vendor ? [ $cwd . '/vendor/autoload.php' ] : [] as $path ) { $real = realpath( $path ); if ( false === $real || 0 === strpos( $real . '/', $cwd . '/' ) ) { continue; @@ -282,6 +395,10 @@ function wp_cli_tests_docker_mounts( $base_dir ) { $mounts = array_values( array_unique( $mounts ) ); + if ( null !== $vendor ) { + $mounts[ $vendor ] = $cwd . '/vendor'; + } + // Download WordPress once for all containers, which only ever copy from it. $version = getenv( 'WP_VERSION' ); if ( $version && ! getenv( 'WP_CLI_TEST_CORE_ZIP' ) ) { @@ -320,7 +437,7 @@ function wp_cli_tests_docker_mounts( $base_dir ) { * @param string|null $db_image Null when running on SQLite. * @param array $mounts * @param array $user_args - * @param string $scenario_list File listing the worker's scenarios. + * @param string|null $scenario_list File listing the worker's scenarios, or null to run those the arguments select. * @return string */ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array $mounts, array $user_args, $scenario_list ) { @@ -367,10 +484,13 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array $command .= ' -e ' . escapeshellarg( "{$key}={$value}" ); } + // This package's own scripts are not linked into vendor/bin when it is the project under test. + $bin = is_file( getcwd() . '/vendor/bin/run-behat-tests' ) ? 'vendor/bin' : 'bin'; + $inner = 'mkdir -p "$HOME"' // Wait for the database server, which takes a few seconds to start, before preparing it. - . ( null !== $db_image ? ' && for i in $(seq 1 60); do mysql -h127.0.0.1 -P3306 --protocol=tcp -uroot -proot -e "SELECT 1" > /dev/null 2>&1 && break; sleep 1; done && vendor/bin/install-package-tests > /dev/null' : '' ) - . ' && vendor/bin/run-behat-tests ' . implode( ' ', array_map( 'escapeshellarg', array_merge( $user_args, [ $scenario_list ] ) ) ); + . ( null !== $db_image ? ' && for i in $(seq 1 60); do mysql -h127.0.0.1 -P3306 --protocol=tcp -uroot -proot -e "SELECT 1" > /dev/null 2>&1 && break; sleep 1; done && ' . $bin . '/install-package-tests > /dev/null' : '' ) + . " && {$bin}/run-behat-tests " . implode( ' ', array_map( 'escapeshellarg', array_merge( $user_args, null === $scenario_list ? [] : [ $scenario_list ] ) ) ); return $command . ' ' . escapeshellarg( $php_image ) . ' sh -c ' . escapeshellarg( $inner ); } @@ -385,7 +505,11 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array $all_args = array_merge( $behat_args, $user_args ); // Run every worker in its own throwaway containers, see run-behat-tests. -$docker = (bool) getenv( 'WP_CLI_TEST_DOCKER' ) && DIRECTORY_SEPARATOR !== '\\'; +$docker = '' !== wp_cli_tests_docker_mode(); +if ( $docker && DIRECTORY_SEPARATOR === '\\' ) { + wp_cli_tests_without_docker( 'The scenarios cannot run in Docker containers on Windows.' ); + $docker = false; +} $rerun_paths = null; // A rerun after a parallel run covers exactly the scenarios that failed in it. @@ -407,32 +531,72 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array } // Paths, name filters and reruns select their own scenarios, so run them as they are. +// Options that take a value as the next argument have to be told apart from paths. $selects_scenarios = false; +$takes_value = false; foreach ( $user_args as $arg ) { - if ( '' === $arg || '-' !== $arg[0] || in_array( $arg, [ '--rerun', '--rerun-only' ], true ) || 0 === strpos( $arg, '--name' ) || '-n' === $arg ) { + if ( $takes_value ) { + $takes_value = false; + continue; + } + if ( in_array( $arg, [ '--format', '-f', '--out', '-o', '--format-settings', '--tags', '--role', '--suite', '-s', '--profile', '-p', '--config', '-c', '--lang' ], true ) ) { + $takes_value = true; + } elseif ( '' === $arg || '-' !== $arg[0] || in_array( $arg, [ '--rerun', '--rerun-only' ], true ) || 0 === strpos( $arg, '--name' ) || '-n' === $arg ) { $selects_scenarios = true; } } -if ( ( $workers < 2 && ! $docker ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths ) ) { +if ( ( $workers < 2 && ! $docker ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths && ! $docker ) ) { wp_cli_tests_run_behat_and_exit( $all_args ); } $php_image = null; $db_image = null; +$vendor = null; if ( $docker ) { - $php_image = wp_cli_tests_docker_php_image(); - $db_image = 'sqlite' === getenv( 'WP_CLI_TEST_DBTYPE' ) ? null : wp_cli_tests_docker_db_image(); + $php_version = wp_cli_tests_docker_php_version(); + $php_image = wp_cli_tests_docker_php_image( $php_version ); + $db_image = 'sqlite' === getenv( 'WP_CLI_TEST_DBTYPE' ) ? null : wp_cli_tests_docker_db_image(); + + if ( null !== $php_image && PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION !== $php_version ) { + $vendor = wp_cli_tests_docker_vendor( $php_image, $php_version ); + if ( null === $vendor ) { + $php_image = null; + } + } + + if ( null === $php_image ) { + wp_cli_tests_without_docker( 'The containers could not be prepared.' ); + } elseif ( null === $db_image && 'sqlite' !== getenv( 'WP_CLI_TEST_DBTYPE' ) ) { + wp_cli_tests_without_docker( 'The type and version of the database server could not be identified, set WP_CLI_TEST_DOCKER_DB to its Docker image.' ); + } if ( null === $php_image || ( null === $db_image && 'sqlite' !== getenv( 'WP_CLI_TEST_DBTYPE' ) ) ) { - fwrite( STDERR, "Warning: Running the scenarios without Docker.\n" ); $docker = false; - if ( $workers < 2 || null !== $rerun_paths ) { + if ( $workers < 2 || null !== $rerun_paths || $selects_scenarios ) { wp_cli_tests_run_behat_and_exit( $all_args ); } } } +// Kept between runs, so that the workers' caches stay warm and failed scenarios can be inspected. +$base_dir = sys_get_temp_dir() . '/wp-cli-test-parallel-' . substr( md5( (string) getcwd() ), 0, 8 ); + +// Scenarios selected through the arguments run in a single container, which selects them itself. +if ( $docker && $selects_scenarios && null === $rerun_paths ) { + if ( ! is_dir( $base_dir ) ) { + mkdir( $base_dir, 0777, true ); + } + $name = 'wp-cli-tests-' . substr( md5( $base_dir ), 0, 8 ) . '-0'; + register_shutdown_function( + static function () use ( $name ) { + exec( 'docker rm -f ' . escapeshellarg( $name ) . ' ' . escapeshellarg( "{$name}-db" ) . ' > /dev/null 2>&1' ); + } + ); + passthru( wp_cli_tests_docker_worker_command( $name, (string) $php_image, $db_image, wp_cli_tests_docker_mounts( $base_dir, $vendor ), $user_args, null ), $exit_code ); + exit( $exit_code ); +} + require getcwd() . '/vendor/autoload.php'; $tags = ''; @@ -442,9 +606,9 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array } } -// Containers check for PHP extensions themselves, as theirs can differ from the host's. +// Containers check for PHP extensions and versions themselves, as theirs can differ from the host's. if ( $docker ) { - $tags = (string) preg_replace( '/(^|&&)~@require-extension-[A-Za-z0-9_]+/', '', $tags ); + $tags = (string) preg_replace( '/(^|&&)~@(require-extension-[A-Za-z0-9_]+|(require|less-than)-php-[0-9.]+)/', '', $tags ); $tags = trim( $tags, '&' ); } @@ -479,10 +643,8 @@ static function ( $path ) { } } -$shards = wp_cli_tests_distribute( $scenarios, $workers ); -// Kept between runs, so that the workers' caches stay warm and failed scenarios can be inspected. -$base_dir = sys_get_temp_dir() . '/wp-cli-test-parallel-' . substr( md5( (string) getcwd() ), 0, 8 ); -$db_name = getenv( 'WP_CLI_TEST_DBNAME' ) ?: 'wp_cli_test'; +$shards = wp_cli_tests_distribute( $scenarios, $workers ); +$db_name = getenv( 'WP_CLI_TEST_DBNAME' ) ?: 'wp_cli_test'; if ( ! is_dir( $base_dir ) ) { mkdir( $base_dir, 0777, true ); @@ -492,7 +654,7 @@ static function ( $path ) { $containers = []; if ( $docker ) { - $mounts = wp_cli_tests_docker_mounts( $base_dir ); + $mounts = wp_cli_tests_docker_mounts( $base_dir, $vendor ); register_shutdown_function( static function () use ( &$containers ) { foreach ( $containers as $container ) { From 282b410ee6bb69881083e84f1afbe8a3aaa08901 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Sat, 31 Oct 2026 19:00:17 +0100 Subject: [PATCH 5/5] Point out that Docker is not used by default Co-Authored-By: Claude Opus 5.5 --- .readme-partials/USING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index f6201a47..f9857143 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -270,11 +270,11 @@ Set `WP_CLI_TEST_DOCKER` to run the scenarios in throwaway Docker containers ins WP_CLI_TEST_DOCKER=auto WP_CLI_TEST_PARALLEL=4 composer behat ``` -| `WP_CLI_TEST_DOCKER` | Effect | -|---------------------------|--------------------------------------------------------------------------------------------------| -| not set, `0`, `no`, `off` | The scenarios run without Docker. | -| `auto` | The scenarios run in containers where possible, and without Docker otherwise, which is reported. | -| `1`, `yes`, `on` | The scenarios run in containers, and the run fails where that is not possible. | +| `WP_CLI_TEST_DOCKER` | Effect | +|-------------------------------------|--------------------------------------------------------------------------------------------------| +| not set (default), `0`, `no`, `off` | The scenarios run without Docker. | +| `auto` | The scenarios run in containers where possible, and without Docker otherwise, which is reported. | +| `1`, `yes`, `on` | The scenarios run in containers, and the run fails where that is not possible. | Docker is not used on Windows, when collecting code coverage, when Docker is not available, when the image cannot be built or when the database server cannot be identified.