diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 6b070a72..e67173e9 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -351,6 +351,17 @@ they test. container and is only reachable over TCP. Set `WP_CLI_TEST_DBSOCKET` to point at the socket if it lives somewhere unusual. +Before the scenarios run, the scenarios that are left out are listed together with the reason, +separating the ones the environment cannot run, such as those needing a PHP extension that is +not loaded, from the ones that are not part of the configuration being tested, such as those +for another database or WordPress version. On GitHub Actions, the list is also added to the +job summary. + +Set `WP_CLI_TEST_STRICT_REQUIREMENTS` where the environment is expected to provide everything +the scenarios need, such as on CI. The run then stops before any scenario starts when some of +them cannot run in the environment, or when the database cannot be reached, instead of leaving +those scenarios out or falling back to SQLite. + #### WP-CLI Binary You can run the tests against a specific WP-CLI binary, instead of using the one that has been built in your project's `vendor/bin` folder. diff --git a/bin/run-behat-tests b/bin/run-behat-tests index 6a9d3d4e..18f8c0b3 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -41,6 +41,10 @@ else if [ -z "${DB_CLIENT}" ]; then echo "Warning: Could not find 'mysql' or 'mariadb' client." + if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then + echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set." + exit 1 + fi echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite." export WP_CLI_TEST_DBTYPE=sqlite else @@ -78,11 +82,15 @@ else DBNAME=${WP_CLI_TEST_DBNAME:-wp_cli_test} - # We need to test the connection. - # Let's try to connect to the specific test database. - if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="USE \`${DBNAME}\`;" 2>/dev/null; then + # We need to test the connection, and that the test user may create the test database. + # The database itself does not need to exist, as the scenarios drop and create it. + if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}\`;" 2>/dev/null; then echo "Warning: Could not connect to the MySQL/MariaDB database." echo "Please make sure the database is running and run 'composer prepare-tests' once to set it up." + if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then + echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set." + exit 1 + fi echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite." export WP_CLI_TEST_DBTYPE=sqlite fi @@ -249,7 +257,8 @@ WP_CLI_TESTS_ROOT="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" export WP_CLI_TESTS_ROOT # Generate the tags to apply environment-specific filters. -BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php) +# This also reports the scenarios that are left out, and stops when strict requirements are not met. +BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php) || exit $? BEHAT_EXTRA_ARGS=() if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/null | grep -q 'xdebug'; then diff --git a/tests/tests/TestBehatTags.php b/tests/tests/TestBehatTags.php index 0ff7f3d2..00e7792e 100644 --- a/tests/tests/TestBehatTags.php +++ b/tests/tests/TestBehatTags.php @@ -573,4 +573,87 @@ public function test_unused_tags_are_not_filtered_out(): void { $this->assertStringNotContainsString( '@require-wp-stable', (string) $output ); $this->assertStringNotContainsString( '@require-mysql-socket', (string) $output ); } + + /** + * Run the script and capture its output streams and exit code. + * + * @param array $env Additional environment variables. + * @return array{stdout: string, stderr: string, exit_code: int} + */ + private function run_behat_tags_script_with_streams( array $env = array() ): array { + $behat_tags = dirname( dirname( __DIR__ ) ) . DIRECTORY_SEPARATOR . 'utils' . DIRECTORY_SEPARATOR . 'behat-tags.php'; + $command = escapeshellarg( PHP_BINARY ) . ' -n ' . escapeshellarg( $behat_tags ); + + $env = array_merge( + getenv(), + array( 'BEHAT_FEATURES_FOLDER' => $this->temp_dir . DIRECTORY_SEPARATOR . 'features' ), + $env + ); + + $process = proc_open( + $command, + array( + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ), + $pipes, + $this->temp_dir, + $env + ); + $this->assertIsResource( $process ); + + $stdout = (string) stream_get_contents( $pipes[1] ); + $stderr = (string) stream_get_contents( $pipes[2] ); + fclose( $pipes[1] ); + fclose( $pipes[2] ); + + return array( + 'stdout' => $stdout, + 'stderr' => $stderr, + 'exit_code' => proc_close( $process ), + ); + } + + public function test_filtered_scenarios_are_reported(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'report.feature', + "Feature: Report\n\n @require-extension-wpclinonexistent\n Scenario: Needs an extension\n Given a step\n\n @broken\n Scenario: Broken\n Given a step\n\n Scenario: Runs\n Given a step\n \"\"\"\n Scenario: Not a scenario of this file\n \"\"\"\n" + ); + + $result = $this->run_behat_tags_script_with_streams(); + + $this->assertSame( 0, $result['exit_code'] ); + $this->assertStringContainsString( '~@require-extension-wpclinonexistent', $result['stdout'] ); + $this->assertStringContainsString( 'Not running 2 of 3 scenarios:', $result['stderr'] ); + $this->assertStringContainsString( 'Not available in this environment:', $result['stderr'] ); + $this->assertStringContainsString( '@require-extension-wpclinonexistent, as the PHP extension "wpclinonexistent" is not loaded: ', $result['stderr'] ); + $this->assertStringContainsString( 'report.feature:4', $result['stderr'] ); + $this->assertStringContainsString( 'Not part of this configuration:', $result['stderr'] ); + $this->assertStringContainsString( 'report.feature:8', $result['stderr'] ); + } + + public function test_strict_requirements_stop_before_the_run(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature', + "Feature: Strict\n\n @require-extension-wpclinonexistent\n Scenario: Needs an extension\n Given a step\n" + ); + + $result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) ); + + $this->assertSame( 1, $result['exit_code'] ); + $this->assertSame( '', $result['stdout'] ); + $this->assertStringContainsString( 'Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and 1 scenarios cannot run in this environment.', $result['stderr'] ); + } + + public function test_strict_requirements_allow_other_configurations(): void { + file_put_contents( + $this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature', + "Feature: Strict\n\n @broken\n Scenario: Broken\n Given a step\n" + ); + + $result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) ); + + $this->assertSame( 0, $result['exit_code'] ); + $this->assertStringContainsString( '~@broken', $result['stdout'] ); + } } diff --git a/utils/behat-tags.php b/utils/behat-tags.php index a17d7c05..cb8365ba 100644 --- a/utils/behat-tags.php +++ b/utils/behat-tags.php @@ -384,9 +384,214 @@ function os_tags( $features_folder = 'features' ) { return $skip_tags; } +/** + * Get the scenarios of the feature files with the tags that apply to them. + * + * @param string $features_folder The folder where the feature files are located. + * @return array}> + */ +function scenarios_with_tags( $features_folder = 'features' ) { + $scenarios = array(); + $feature_files = glob( $features_folder . DIRECTORY_SEPARATOR . '*.feature' ); + + foreach ( $feature_files ? $feature_files : array() as $feature_file ) { + $lines = file( $feature_file, FILE_IGNORE_NEW_LINES ); + $feature_tags = array(); + $pending_tags = array(); + $doc_string = null; + + foreach ( $lines ? $lines : array() as $index => $line ) { + $line = trim( $line ); + + // Doc strings can hold whole feature files, as written by scenarios for scaffolding. + if ( null !== $doc_string ) { + if ( 0 === strpos( $line, $doc_string ) ) { + $doc_string = null; + } + continue; + } + if ( 0 === strpos( $line, '"""' ) || 0 === strpos( $line, '```' ) ) { + $doc_string = substr( $line, 0, 3 ); + continue; + } + + if ( 0 === strpos( $line, '@' ) ) { + preg_match_all( '/@[^\s@]+/', $line, $matches ); + $pending_tags = array_merge( $pending_tags, $matches[0] ); + } elseif ( preg_match( '/^Feature:/', $line ) ) { + $feature_tags = $pending_tags; + $pending_tags = array(); + } elseif ( preg_match( '/^(Scenario|Scenario Outline|Scenario Template|Example):/', $line ) ) { + $scenarios[] = array( + 'path' => $feature_file . ':' . ( $index + 1 ), + 'tags' => array_merge( $feature_tags, $pending_tags ), + ); + $pending_tags = array(); + } elseif ( '' !== $line && 0 !== strpos( $line, '#' ) ) { + $pending_tags = array(); + } + } + } + + return $scenarios; +} + +/** + * Describe why scenarios carrying a skip tag are not run. + * + * @param string $tag The tag, without the leading tilde. + * @param array $context What the environment provides: wp, php, db and os. + * @return array{0: bool, 1: string} Whether the environment lacks something, and the reason. + */ +function describe_skip_tag( $tag, $context ) { + if ( preg_match( '/^@require-extension-(.+)$/', $tag, $matches ) ) { + return array( true, "the PHP extension \"{$matches[1]}\" is not loaded" ); + } + + if ( '@require-mysql-socket' === $tag ) { + return array( true, 'the database server cannot be reached through a socket' ); + } + + if ( preg_match( '/^@(require|less-than)-wp-/', $tag ) || '@require-wp-stable' === $tag || '@broken-trunk' === $tag ) { + return array( false, "WordPress {$context['wp']} is used" ); + } + + if ( preg_match( '/^@(require|less-than)-php-/', $tag ) ) { + return array( false, "PHP {$context['php']} is used" ); + } + + if ( preg_match( '/^@(require|less-than|skip)-(mysql|mariadb|sqlite)/', $tag ) ) { + return array( false, "the database is {$context['db']}" ); + } + + if ( preg_match( '/^@(require|skip)-(windows|macos|linux)$/', $tag ) ) { + return array( false, "the operating system is {$context['os']}" ); + } + + if ( '@github-api' === $tag ) { + return array( false, 'GITHUB_TOKEN is not set' ); + } + + if ( '@require-object-cache' === $tag || '@skip-object-cache' === $tag ) { + return array( false, 'WP_CLI_TEST_OBJECT_CACHE is ' . ( getenv( 'WP_CLI_TEST_OBJECT_CACHE' ) ? 'set' : 'not set' ) ); + } + + if ( '@broken' === $tag ) { + return array( false, 'they are known to be broken' ); + } + + return array( false, 'it is excluded' ); +} + +/** + * Report the scenarios that the tag filter leaves out, and why. + * + * Scenarios that the environment cannot run are listed separately from the ones that are + * not part of the configuration being tested, such as another database or WordPress version. + * + * @param array $skip_tags + * @param string $features_folder + * @param array $context + * @return int Number of scenarios left out because the environment lacks something. + */ +function report_filtered_scenarios( $skip_tags, $features_folder, $context ) { + $scenarios = scenarios_with_tags( $features_folder ); + $groups = array(); + + foreach ( $scenarios as $scenario ) { + $matching = array_values( array_intersect( $skip_tags, $scenario['tags'] ) ); + if ( $matching ) { + $groups[ $matching[0] ][] = $scenario['path']; + } + } + + if ( ! $groups ) { + return 0; + } + + $sections = array( + true => array(), + false => array(), + ); + $missing = 0; + $total = 0; + + foreach ( $groups as $tag => $paths ) { + list( $lacks, $reason ) = describe_skip_tag( $tag, $context ); + $sections[ $lacks ][] = array( $tag, $reason, $paths ); + $total += count( $paths ); + if ( $lacks ) { + $missing += count( $paths ); + } + } + + $titles = array( + true => 'Not available in this environment', + false => 'Not part of this configuration', + ); + + $text = sprintf( "Not running %d of %d scenarios:\n", $total, count( $scenarios ) ); + $markdown = sprintf( "### Behat: %d of %d scenarios not run\n", $total, count( $scenarios ) ); + + foreach ( array( true, false ) as $lacks ) { + if ( ! $sections[ $lacks ] ) { + continue; + } + $text .= " {$titles[ $lacks ]}:\n"; + $markdown .= "\n**{$titles[ $lacks ]}**\n\n"; + foreach ( $sections[ $lacks ] as $section ) { + list( $tag, $reason, $paths ) = $section; + + $text .= sprintf( " %s, as %s: %s\n", $tag, $reason, implode( ', ', $paths ) ); + $markdown .= sprintf( + "- `%s`, as %s (%d): %s\n", + $tag, + $reason, + count( $paths ), + implode( + ', ', + array_map( + function ( $path ) { + return "`{$path}`"; + }, + $paths + ) + ) + ); + } + } + + fwrite( STDERR, $text ); + + // Add the report to the summary of a GitHub Actions job, once per run. + $summary = getenv( 'GITHUB_STEP_SUMMARY' ); + if ( $summary && ! getenv( 'WP_CLI_TEST_DEBUG_BEHAT_ENV' ) ) { + file_put_contents( $summary, $markdown . "\n", FILE_APPEND ); + } + + return $missing; +} + $skip_tags = array_merge( $skip_tags, extension_tags( $features_folder ) ); $skip_tags = array_merge( $skip_tags, os_tags( $features_folder ) ); +$missing = report_filtered_scenarios( + $skip_tags, + $features_folder, + array( + 'wp' => $wp_version ? $wp_version : 'latest', + 'php' => PHP_VERSION, + 'db' => trim( ( 'sqlite' === $db_type ? 'SQLite' : ( 'mariadb' === $db_type ? 'MariaDB' : 'MySQL' ) ) . ' ' . ( 'sqlite' === $db_type ? '' : $db_version ) ), + 'os' => PHP_OS_FAMILY, + ) +); + +// With strict requirements, everything the scenarios need has to be available. +if ( $missing && getenv( 'WP_CLI_TEST_STRICT_REQUIREMENTS' ) ) { + fwrite( STDERR, "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and {$missing} scenarios cannot run in this environment.\n" ); + exit( 1 ); +} + if ( ! empty( $skip_tags ) ) { echo '--tags=~' . implode( '&&~', $skip_tags ); }