From f9a9003241b36d82fe6f90f2a1e8e1a184be9846 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Fri, 30 Oct 2026 21:07:41 +0100 Subject: [PATCH] Report the scenarios that are left out, and why Scenarios the environment cannot run, or that belong to another configuration, were filtered out without a word. List them before the run with the reason, separating what the environment lacks from what is not part of the configuration being tested, and add the list to the job summary on GitHub Actions. With WP_CLI_TEST_STRICT_REQUIREMENTS set, the run stops before any scenario starts when the environment lacks something, or when the database cannot be reached, instead of leaving scenarios out or falling back to SQLite. The database check no longer needs the test database to exist, which the scenarios drop themselves, so that a run no longer falls back to SQLite just because the previous one left no test database behind. Co-Authored-By: Claude Opus 5.5 --- .readme-partials/USING.md | 11 ++ bin/run-behat-tests | 17 ++- tests/tests/TestBehatTags.php | 83 ++++++++++++++ utils/behat-tags.php | 205 ++++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 4 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 6b070a72..e67173e9 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -351,6 +351,17 @@ they test. container and is only reachable over TCP. Set `WP_CLI_TEST_DBSOCKET` to point at the socket if it lives somewhere unusual. +Before the scenarios run, the scenarios that are left out are listed together with the reason, +separating the ones the environment cannot run, such as those needing a PHP extension that is +not loaded, from the ones that are not part of the configuration being tested, such as those +for another database or WordPress version. On GitHub Actions, the list is also added to the +job summary. + +Set `WP_CLI_TEST_STRICT_REQUIREMENTS` where the environment is expected to provide everything +the scenarios need, such as on CI. The run then stops before any scenario starts when some of +them cannot run in the environment, or when the database cannot be reached, instead of leaving +those scenarios out or falling back to SQLite. + #### WP-CLI Binary You can run the tests against a specific WP-CLI binary, instead of using the one that has been built in your project's `vendor/bin` folder. diff --git a/bin/run-behat-tests b/bin/run-behat-tests index 6a9d3d4e..18f8c0b3 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -41,6 +41,10 @@ else if [ -z "${DB_CLIENT}" ]; then echo "Warning: Could not find 'mysql' or 'mariadb' client." + if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then + echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set." + exit 1 + fi echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite." export WP_CLI_TEST_DBTYPE=sqlite else @@ -78,11 +82,15 @@ else DBNAME=${WP_CLI_TEST_DBNAME:-wp_cli_test} - # We need to test the connection. - # Let's try to connect to the specific test database. - if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="USE \`${DBNAME}\`;" 2>/dev/null; then + # We need to test the connection, and that the test user may create the test database. + # The database itself does not need to exist, as the scenarios drop and create it. + if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}\`;" 2>/dev/null; then echo "Warning: Could not connect to the MySQL/MariaDB database." echo "Please make sure the database is running and run 'composer prepare-tests' once to set it up." + if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then + echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set." + exit 1 + fi echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite." export WP_CLI_TEST_DBTYPE=sqlite fi @@ -249,7 +257,8 @@ WP_CLI_TESTS_ROOT="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" export WP_CLI_TESTS_ROOT # Generate the tags to apply environment-specific filters. -BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php) +# This also reports the scenarios that are left out, and stops when strict requirements are not met. +BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php) || exit $? BEHAT_EXTRA_ARGS=() if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/null | grep -q 'xdebug'; then diff --git a/tests/tests/TestBehatTags.php b/tests/tests/TestBehatTags.php index 0ff7f3d2..00e7792e 100644 --- a/tests/tests/TestBehatTags.php +++ b/tests/tests/TestBehatTags.php @@ -573,4 +573,87 @@ public function test_unused_tags_are_not_filtered_out(): void { $this->assertStringNotContainsString( '@require-wp-stable', (string) $output ); $this->assertStringNotContainsString( '@require-mysql-socket', (string) $output ); } + + /** + * Run the script and capture its output streams and exit code. + * + * @param array $env Additional environment variables. + * @return array{stdout: string, stderr: string, exit_code: int} + */ + private function run_behat_tags_script_with_streams( array $env = array() ): array { + $behat_tags = dirname( dirname( __DIR__ ) ) . DIRECTORY_SEPARATOR . 'utils' . DIRECTORY_SEPARATOR . 'behat-tags.php'; + $command = escapeshellarg( PHP_BINARY ) . ' -n ' . escapeshellarg( $behat_tags ); + + $env = array_merge( + getenv(), + array( 'BEHAT_FEATURES_FOLDER' => $this->temp_dir . DIRECTORY_SEPARATOR . 'features' ), + $env + ); + + $process = proc_open( + $command, + array( + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ), + $pipes, + $this->temp_dir, + $env + ); + $this->assertIsResource( $process ); + + $stdout = (string) stream_get_contents( $pipes[1] ); + $stderr = (string) stream_get_contents( $pipes[2] ); + fclose( $pipes[1] ); + fclose( $pipes[2] ); + + return array( + 'stdout' => $stdout, + 'stderr' => $stderr, + 'exit_code' => proc_close( $process ), + ); + } + + public function test_filtered_scenarios_are_reported(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'report.feature', + "Feature: Report\n\n @require-extension-wpclinonexistent\n Scenario: Needs an extension\n Given a step\n\n @broken\n Scenario: Broken\n Given a step\n\n Scenario: Runs\n Given a step\n \"\"\"\n Scenario: Not a scenario of this file\n \"\"\"\n" + ); + + $result = $this->run_behat_tags_script_with_streams(); + + $this->assertSame( 0, $result['exit_code'] ); + $this->assertStringContainsString( '~@require-extension-wpclinonexistent', $result['stdout'] ); + $this->assertStringContainsString( 'Not running 2 of 3 scenarios:', $result['stderr'] ); + $this->assertStringContainsString( 'Not available in this environment:', $result['stderr'] ); + $this->assertStringContainsString( '@require-extension-wpclinonexistent, as the PHP extension "wpclinonexistent" is not loaded: ', $result['stderr'] ); + $this->assertStringContainsString( 'report.feature:4', $result['stderr'] ); + $this->assertStringContainsString( 'Not part of this configuration:', $result['stderr'] ); + $this->assertStringContainsString( 'report.feature:8', $result['stderr'] ); + } + + public function test_strict_requirements_stop_before_the_run(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature', + "Feature: Strict\n\n @require-extension-wpclinonexistent\n Scenario: Needs an extension\n Given a step\n" + ); + + $result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) ); + + $this->assertSame( 1, $result['exit_code'] ); + $this->assertSame( '', $result['stdout'] ); + $this->assertStringContainsString( 'Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and 1 scenarios cannot run in this environment.', $result['stderr'] ); + } + + public function test_strict_requirements_allow_other_configurations(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature', + "Feature: Strict\n\n @broken\n Scenario: Broken\n Given a step\n" + ); + + $result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) ); + + $this->assertSame( 0, $result['exit_code'] ); + $this->assertStringContainsString( '~@broken', $result['stdout'] ); + } } diff --git a/utils/behat-tags.php b/utils/behat-tags.php index a17d7c05..cb8365ba 100644 --- a/utils/behat-tags.php +++ b/utils/behat-tags.php @@ -384,9 +384,214 @@ function os_tags( $features_folder = 'features' ) { return $skip_tags; } +/** + * Get the scenarios of the feature files with the tags that apply to them. + * + * @param string $features_folder The folder where the feature files are located. + * @return array}> + */ +function scenarios_with_tags( $features_folder = 'features' ) { + $scenarios = array(); + $feature_files = glob( $features_folder . DIRECTORY_SEPARATOR . '*.feature' ); + + foreach ( $feature_files ? $feature_files : array() as $feature_file ) { + $lines = file( $feature_file, FILE_IGNORE_NEW_LINES ); + $feature_tags = array(); + $pending_tags = array(); + $doc_string = null; + + foreach ( $lines ? $lines : array() as $index => $line ) { + $line = trim( $line ); + + // Doc strings can hold whole feature files, as written by scenarios for scaffolding. + if ( null !== $doc_string ) { + if ( 0 === strpos( $line, $doc_string ) ) { + $doc_string = null; + } + continue; + } + if ( 0 === strpos( $line, '"""' ) || 0 === strpos( $line, '```' ) ) { + $doc_string = substr( $line, 0, 3 ); + continue; + } + + if ( 0 === strpos( $line, '@' ) ) { + preg_match_all( '/@[^\s@]+/', $line, $matches ); + $pending_tags = array_merge( $pending_tags, $matches[0] ); + } elseif ( preg_match( '/^Feature:/', $line ) ) { + $feature_tags = $pending_tags; + $pending_tags = array(); + } elseif ( preg_match( '/^(Scenario|Scenario Outline|Scenario Template|Example):/', $line ) ) { + $scenarios[] = array( + 'path' => $feature_file . ':' . ( $index + 1 ), + 'tags' => array_merge( $feature_tags, $pending_tags ), + ); + $pending_tags = array(); + } elseif ( '' !== $line && 0 !== strpos( $line, '#' ) ) { + $pending_tags = array(); + } + } + } + + return $scenarios; +} + +/** + * Describe why scenarios carrying a skip tag are not run. + * + * @param string $tag The tag, without the leading tilde. + * @param array $context What the environment provides: wp, php, db and os. + * @return array{0: bool, 1: string} Whether the environment lacks something, and the reason. + */ +function describe_skip_tag( $tag, $context ) { + if ( preg_match( '/^@require-extension-(.+)$/', $tag, $matches ) ) { + return array( true, "the PHP extension \"{$matches[1]}\" is not loaded" ); + } + + if ( '@require-mysql-socket' === $tag ) { + return array( true, 'the database server cannot be reached through a socket' ); + } + + if ( preg_match( '/^@(require|less-than)-wp-/', $tag ) || '@require-wp-stable' === $tag || '@broken-trunk' === $tag ) { + return array( false, "WordPress {$context['wp']} is used" ); + } + + if ( preg_match( '/^@(require|less-than)-php-/', $tag ) ) { + return array( false, "PHP {$context['php']} is used" ); + } + + if ( preg_match( '/^@(require|less-than|skip)-(mysql|mariadb|sqlite)/', $tag ) ) { + return array( false, "the database is {$context['db']}" ); + } + + if ( preg_match( '/^@(require|skip)-(windows|macos|linux)$/', $tag ) ) { + return array( false, "the operating system is {$context['os']}" ); + } + + if ( '@github-api' === $tag ) { + return array( false, 'GITHUB_TOKEN is not set' ); + } + + if ( '@require-object-cache' === $tag || '@skip-object-cache' === $tag ) { + return array( false, 'WP_CLI_TEST_OBJECT_CACHE is ' . ( getenv( 'WP_CLI_TEST_OBJECT_CACHE' ) ? 'set' : 'not set' ) ); + } + + if ( '@broken' === $tag ) { + return array( false, 'they are known to be broken' ); + } + + return array( false, 'it is excluded' ); +} + +/** + * Report the scenarios that the tag filter leaves out, and why. + * + * Scenarios that the environment cannot run are listed separately from the ones that are + * not part of the configuration being tested, such as another database or WordPress version. + * + * @param array $skip_tags + * @param string $features_folder + * @param array $context + * @return int Number of scenarios left out because the environment lacks something. + */ +function report_filtered_scenarios( $skip_tags, $features_folder, $context ) { + $scenarios = scenarios_with_tags( $features_folder ); + $groups = array(); + + foreach ( $scenarios as $scenario ) { + $matching = array_values( array_intersect( $skip_tags, $scenario['tags'] ) ); + if ( $matching ) { + $groups[ $matching[0] ][] = $scenario['path']; + } + } + + if ( ! $groups ) { + return 0; + } + + $sections = array( + true => array(), + false => array(), + ); + $missing = 0; + $total = 0; + + foreach ( $groups as $tag => $paths ) { + list( $lacks, $reason ) = describe_skip_tag( $tag, $context ); + $sections[ $lacks ][] = array( $tag, $reason, $paths ); + $total += count( $paths ); + if ( $lacks ) { + $missing += count( $paths ); + } + } + + $titles = array( + true => 'Not available in this environment', + false => 'Not part of this configuration', + ); + + $text = sprintf( "Not running %d of %d scenarios:\n", $total, count( $scenarios ) ); + $markdown = sprintf( "### Behat: %d of %d scenarios not run\n", $total, count( $scenarios ) ); + + foreach ( array( true, false ) as $lacks ) { + if ( ! $sections[ $lacks ] ) { + continue; + } + $text .= " {$titles[ $lacks ]}:\n"; + $markdown .= "\n**{$titles[ $lacks ]}**\n\n"; + foreach ( $sections[ $lacks ] as $section ) { + list( $tag, $reason, $paths ) = $section; + + $text .= sprintf( " %s, as %s: %s\n", $tag, $reason, implode( ', ', $paths ) ); + $markdown .= sprintf( + "- `%s`, as %s (%d): %s\n", + $tag, + $reason, + count( $paths ), + implode( + ', ', + array_map( + function ( $path ) { + return "`{$path}`"; + }, + $paths + ) + ) + ); + } + } + + fwrite( STDERR, $text ); + + // Add the report to the summary of a GitHub Actions job, once per run. + $summary = getenv( 'GITHUB_STEP_SUMMARY' ); + if ( $summary && ! getenv( 'WP_CLI_TEST_DEBUG_BEHAT_ENV' ) ) { + file_put_contents( $summary, $markdown . "\n", FILE_APPEND ); + } + + return $missing; +} + $skip_tags = array_merge( $skip_tags, extension_tags( $features_folder ) ); $skip_tags = array_merge( $skip_tags, os_tags( $features_folder ) ); +$missing = report_filtered_scenarios( + $skip_tags, + $features_folder, + array( + 'wp' => $wp_version ? $wp_version : 'latest', + 'php' => PHP_VERSION, + 'db' => trim( ( 'sqlite' === $db_type ? 'SQLite' : ( 'mariadb' === $db_type ? 'MariaDB' : 'MySQL' ) ) . ' ' . ( 'sqlite' === $db_type ? '' : $db_version ) ), + 'os' => PHP_OS_FAMILY, + ) +); + +// With strict requirements, everything the scenarios need has to be available. +if ( $missing && getenv( 'WP_CLI_TEST_STRICT_REQUIREMENTS' ) ) { + fwrite( STDERR, "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and {$missing} scenarios cannot run in this environment.\n" ); + exit( 1 ); +} + if ( ! empty( $skip_tags ) ) { echo '--tags=~' . implode( '&&~', $skip_tags ); }