schlessera · Oct 30, 2026
@@ -351,6 +351,17 @@ they test.
   container and is only reachable over TCP. Set `WP_CLI_TEST_DBSOCKET` to point at the
   socket if it lives somewhere unusual.
 
+Before the scenarios run, the scenarios that are left out are listed together with the reason,
+separating the ones the environment cannot run, such as those needing a PHP extension that is
+not loaded, from the ones that are not part of the configuration being tested, such as those
+for another database or WordPress version. On GitHub Actions, the list is also added to the
+job summary.
+
+Set `WP_CLI_TEST_STRICT_REQUIREMENTS` where the environment is expected to provide everything
+the scenarios need, such as on CI. The run then stops before any scenario starts when some of
+them cannot run in the environment, or when the database cannot be reached, instead of leaving
+those scenarios out or falling back to SQLite.
+
 #### WP-CLI Binary
 
 You can run the tests against a specific WP-CLI binary, instead of using the one that has been built in your project's `vendor/bin` folder.

@@ -41,6 +41,10 @@ else
 
 	if [ -z "${DB_CLIENT}" ]; then
 		echo "Warning: Could not find 'mysql' or 'mariadb' client."
+		if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then
+			echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set."
+			exit 1
+		fi
 		echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite."
 		export WP_CLI_TEST_DBTYPE=sqlite
 	else
@@ -78,11 +82,15 @@ else
 
 		DBNAME=${WP_CLI_TEST_DBNAME:-wp_cli_test}
 
-		# We need to test the connection.
-		# Let's try to connect to the specific test database.
-		if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="USE \`${DBNAME}\`;" 2>/dev/null; then
+		# We need to test the connection, and that the test user may create the test database.
+		# The database itself does not need to exist, as the scenarios drop and create it.
+		if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}\`;" 2>/dev/null; then
 			echo "Warning: Could not connect to the MySQL/MariaDB database."
 			echo "Please make sure the database is running and run 'composer prepare-tests' once to set it up."
+			if [ -n "${WP_CLI_TEST_STRICT_REQUIREMENTS-}" ]; then
+				echo "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set."
+				exit 1
+			fi
 			echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite."
 			export WP_CLI_TEST_DBTYPE=sqlite
 		fi
@@ -249,7 +257,8 @@ WP_CLI_TESTS_ROOT="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
 export WP_CLI_TESTS_ROOT
 
 # Generate the tags to apply environment-specific filters.
-BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php)
+# This also reports the scenarios that are left out, and stops when strict requirements are not met.
+BEHAT_TAGS=$(php "$WP_CLI_TESTS_ROOT"/utils/behat-tags.php) || exit $?
 
 BEHAT_EXTRA_ARGS=()
 if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/null | grep -q 'xdebug'; then

@@ -573,4 +573,87 @@ public function test_unused_tags_are_not_filtered_out(): void {
 		$this->assertStringNotContainsString( '@require-wp-stable', (string) $output );
 		$this->assertStringNotContainsString( '@require-mysql-socket', (string) $output );
 	}
+
+	/**
+	 * Run the script and capture its output streams and exit code.
+	 *
+	 * @param array<string, string> $env Additional environment variables.
+	 * @return array{stdout: string, stderr: string, exit_code: int}
+	 */
+	private function run_behat_tags_script_with_streams( array $env = array() ): array {
+		$behat_tags = dirname( dirname( __DIR__ ) ) . DIRECTORY_SEPARATOR . 'utils' . DIRECTORY_SEPARATOR . 'behat-tags.php';
+		$command    = escapeshellarg( PHP_BINARY ) . ' -n ' . escapeshellarg( $behat_tags );
+
+		$env = array_merge(
+			getenv(),
+			array( 'BEHAT_FEATURES_FOLDER' => $this->temp_dir . DIRECTORY_SEPARATOR . 'features' ),
+			$env
+		);
+
+		$process = proc_open(
+			$command,
+			array(
+				1 => array( 'pipe', 'w' ),
+				2 => array( 'pipe', 'w' ),
+			),
+			$pipes,
+			$this->temp_dir,
+			$env
+		);
+		$this->assertIsResource( $process );
+
+		$stdout = (string) stream_get_contents( $pipes[1] );
+		$stderr = (string) stream_get_contents( $pipes[2] );
+		fclose( $pipes[1] );
+		fclose( $pipes[2] );
+
+		return array(
+			'stdout'    => $stdout,
+			'stderr'    => $stderr,
+			'exit_code' => proc_close( $process ),
+		);
+	}
+
+	public function test_filtered_scenarios_are_reported(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'report.feature',
+			"Feature: Report\n\n  @require-extension-wpclinonexistent\n  Scenario: Needs an extension\n    Given a step\n\n  @broken\n  Scenario: Broken\n    Given a step\n\n  Scenario: Runs\n    Given a step\n      \"\"\"\n      Scenario: Not a scenario of this file\n      \"\"\"\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams();
+
+		$this->assertSame( 0, $result['exit_code'] );
+		$this->assertStringContainsString( '~@require-extension-wpclinonexistent', $result['stdout'] );
+		$this->assertStringContainsString( 'Not running 2 of 3 scenarios:', $result['stderr'] );
+		$this->assertStringContainsString( 'Not available in this environment:', $result['stderr'] );
+		$this->assertStringContainsString( '@require-extension-wpclinonexistent, as the PHP extension "wpclinonexistent" is not loaded: ', $result['stderr'] );
+		$this->assertStringContainsString( 'report.feature:4', $result['stderr'] );
+		$this->assertStringContainsString( 'Not part of this configuration:', $result['stderr'] );
+		$this->assertStringContainsString( 'report.feature:8', $result['stderr'] );
+	}
+
+	public function test_strict_requirements_stop_before_the_run(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature',
+			"Feature: Strict\n\n  @require-extension-wpclinonexistent\n  Scenario: Needs an extension\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) );
+
+		$this->assertSame( 1, $result['exit_code'] );
+		$this->assertSame( '', $result['stdout'] );
+		$this->assertStringContainsString( 'Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and 1 scenarios cannot run in this environment.', $result['stderr'] );
+	}
+
+	public function test_strict_requirements_allow_other_configurations(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'strict.feature',
+			"Feature: Strict\n\n  @broken\n  Scenario: Broken\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams( array( 'WP_CLI_TEST_STRICT_REQUIREMENTS' => '1' ) );
+
+		$this->assertSame( 0, $result['exit_code'] );
+		$this->assertStringContainsString( '~@broken', $result['stdout'] );
+	}
 }
@@ -384,9 +384,214 @@ function os_tags( $features_folder = 'features' ) {
 	return $skip_tags;
 }
 
+/**
+ * Get the scenarios of the feature files with the tags that apply to them.
+ *
+ * @param string $features_folder The folder where the feature files are located.
+ * @return array<array{path: string, tags: array<string>}>
+ */
+function scenarios_with_tags( $features_folder = 'features' ) {
+	$scenarios     = array();
+	$feature_files = glob( $features_folder . DIRECTORY_SEPARATOR . '*.feature' );
+
+	foreach ( $feature_files ? $feature_files : array() as $feature_file ) {
+		$lines        = file( $feature_file, FILE_IGNORE_NEW_LINES );
+		$feature_tags = array();
+		$pending_tags = array();
+		$doc_string   = null;
+
+		foreach ( $lines ? $lines : array() as $index => $line ) {
+			$line = trim( $line );
+
+			// Doc strings can hold whole feature files, as written by scenarios for scaffolding.
+			if ( null !== $doc_string ) {
+				if ( 0 === strpos( $line, $doc_string ) ) {
+					$doc_string = null;
+				}
+				continue;
+			}
+			if ( 0 === strpos( $line, '"""' ) || 0 === strpos( $line, '```' ) ) {
+				$doc_string = substr( $line, 0, 3 );
+				continue;
+			}
+
+			if ( 0 === strpos( $line, '@' ) ) {
+				preg_match_all( '/@[^\s@]+/', $line, $matches );
+				$pending_tags = array_merge( $pending_tags, $matches[0] );
+			} elseif ( preg_match( '/^Feature:/', $line ) ) {
+				$feature_tags = $pending_tags;
+				$pending_tags = array();
+			} elseif ( preg_match( '/^(Scenario|Scenario Outline|Scenario Template|Example):/', $line ) ) {
+				$scenarios[]  = array(
+					'path' => $feature_file . ':' . ( $index + 1 ),
+					'tags' => array_merge( $feature_tags, $pending_tags ),
+				);
+				$pending_tags = array();
+			} elseif ( '' !== $line && 0 !== strpos( $line, '#' ) ) {
+				$pending_tags = array();
+			}
+		}
+	}
+
+	return $scenarios;
+}
+
+/**
+ * Describe why scenarios carrying a skip tag are not run.
+ *
+ * @param string $tag     The tag, without the leading tilde.
+ * @param array  $context What the environment provides: wp, php, db and os.
+ * @return array{0: bool, 1: string} Whether the environment lacks something, and the reason.
+ */
+function describe_skip_tag( $tag, $context ) {
+	if ( preg_match( '/^@require-extension-(.+)$/', $tag, $matches ) ) {
+		return array( true, "the PHP extension \"{$matches[1]}\" is not loaded" );
+	}
+
+	if ( '@require-mysql-socket' === $tag ) {
+		return array( true, 'the database server cannot be reached through a socket' );
+	}
+
+	if ( preg_match( '/^@(require|less-than)-wp-/', $tag ) || '@require-wp-stable' === $tag || '@broken-trunk' === $tag ) {
+		return array( false, "WordPress {$context['wp']} is used" );
+	}
+
+	if ( preg_match( '/^@(require|less-than)-php-/', $tag ) ) {
+		return array( false, "PHP {$context['php']} is used" );
+	}
+
+	if ( preg_match( '/^@(require|less-than|skip)-(mysql|mariadb|sqlite)/', $tag ) ) {
+		return array( false, "the database is {$context['db']}" );
+	}
+
+	if ( preg_match( '/^@(require|skip)-(windows|macos|linux)$/', $tag ) ) {
+		return array( false, "the operating system is {$context['os']}" );
+	}
+
+	if ( '@github-api' === $tag ) {
+		return array( false, 'GITHUB_TOKEN is not set' );
+	}
+
+	if ( '@require-object-cache' === $tag || '@skip-object-cache' === $tag ) {
+		return array( false, 'WP_CLI_TEST_OBJECT_CACHE is ' . ( getenv( 'WP_CLI_TEST_OBJECT_CACHE' ) ? 'set' : 'not set' ) );
+	}
+
+	if ( '@broken' === $tag ) {
+		return array( false, 'they are known to be broken' );
+	}
+
+	return array( false, 'it is excluded' );
+}
+
+/**
+ * Report the scenarios that the tag filter leaves out, and why.
+ *
+ * Scenarios that the environment cannot run are listed separately from the ones that are
+ * not part of the configuration being tested, such as another database or WordPress version.
+ *
+ * @param array<string> $skip_tags
+ * @param string        $features_folder
+ * @param array         $context
+ * @return int Number of scenarios left out because the environment lacks something.
+ */
+function report_filtered_scenarios( $skip_tags, $features_folder, $context ) {
+	$scenarios = scenarios_with_tags( $features_folder );
+	$groups    = array();
+
+	foreach ( $scenarios as $scenario ) {
+		$matching = array_values( array_intersect( $skip_tags, $scenario['tags'] ) );
+		if ( $matching ) {
+			$groups[ $matching[0] ][] = $scenario['path'];
+		}
+	}
+
+	if ( ! $groups ) {
+		return 0;
+	}
+
+	$sections = array(
+		true  => array(),
+		false => array(),
+	);
+	$missing  = 0;
+	$total    = 0;
+
+	foreach ( $groups as $tag => $paths ) {
+		list( $lacks, $reason ) = describe_skip_tag( $tag, $context );
+		$sections[ $lacks ][]   = array( $tag, $reason, $paths );
+		$total                 += count( $paths );
+		if ( $lacks ) {
+			$missing += count( $paths );
+		}
+	}
+
+	$titles = array(
+		true  => 'Not available in this environment',
+		false => 'Not part of this configuration',
+	);
+
+	$text     = sprintf( "Not running %d of %d scenarios:\n", $total, count( $scenarios ) );
+	$markdown = sprintf( "### Behat: %d of %d scenarios not run\n", $total, count( $scenarios ) );
+
+	foreach ( array( true, false ) as $lacks ) {
+		if ( ! $sections[ $lacks ] ) {
+			continue;
+		}
+		$text     .= "  {$titles[ $lacks ]}:\n";
+		$markdown .= "\n**{$titles[ $lacks ]}**\n\n";
+		foreach ( $sections[ $lacks ] as $section ) {
+			list( $tag, $reason, $paths ) = $section;
+
+			$text     .= sprintf( "    %s, as %s: %s\n", $tag, $reason, implode( ', ', $paths ) );
+			$markdown .= sprintf(
+				"- `%s`, as %s (%d): %s\n",
+				$tag,
+				$reason,
+				count( $paths ),
+				implode(
+					', ',
+					array_map(
+						function ( $path ) {
+							return "`{$path}`";
+						},
+						$paths
+					)
+				)
+			);
+		}
+	}
+
+	fwrite( STDERR, $text );
+
+	// Add the report to the summary of a GitHub Actions job, once per run.
+	$summary = getenv( 'GITHUB_STEP_SUMMARY' );
+	if ( $summary && ! getenv( 'WP_CLI_TEST_DEBUG_BEHAT_ENV' ) ) {
+		file_put_contents( $summary, $markdown . "\n", FILE_APPEND );
+	}
+
+	return $missing;
+}
+
 $skip_tags = array_merge( $skip_tags, extension_tags( $features_folder ) );
 $skip_tags = array_merge( $skip_tags, os_tags( $features_folder ) );
 
+$missing = report_filtered_scenarios(
+	$skip_tags,
+	$features_folder,
+	array(
+		'wp'  => $wp_version ? $wp_version : 'latest',
+		'php' => PHP_VERSION,
+		'db'  => trim( ( 'sqlite' === $db_type ? 'SQLite' : ( 'mariadb' === $db_type ? 'MariaDB' : 'MySQL' ) ) . ' ' . ( 'sqlite' === $db_type ? '' : $db_version ) ),
+		'os'  => PHP_OS_FAMILY,
+	)
+);
+
+// With strict requirements, everything the scenarios need has to be available.
+if ( $missing && getenv( 'WP_CLI_TEST_STRICT_REQUIREMENTS' ) ) {
+	fwrite( STDERR, "Stopping, as WP_CLI_TEST_STRICT_REQUIREMENTS is set and {$missing} scenarios cannot run in this environment.\n" );
+	exit( 1 );
+}
+
 if ( ! empty( $skip_tags ) ) {
 	echo '--tags=~' . implode( '&&~', $skip_tags );
 }