From 6470794a66ec5c7e483a16d36fbbb6de3ac8c89e Mon Sep 17 00:00:00 2001
From: Alain Schlesser <alain.schlesser@gmail.com>
Date: Fri, 30 Oct 2026 21:14:12 +0100
Subject: [PATCH] Declare and check the commands that scenarios need

Scenarios can declare commands they run that are not always installed with
@require-bin-<name> tags. Where a command is not available, its scenarios
are left out and reported like other unmet requirements, instead of failing
halfway through.

The existing @require-composer, @require-download and
@require-opcache-save-comments tags never filtered anything and are checked
now. Unknown requirement tags are reported, as they would not filter
anything either.

When a command fails because a command it runs does not exist, the failure
names the tag to add, and the Gherkin linting lists scenarios that run
commands which are not always installed without declaring them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .readme-partials/USING.md              |  12 +++
 bin/run-gherkin-lint-tests             |  25 +++--
 phpcs.xml.dist                         |   2 +
 src/Context/Support.php                |  25 +++++
 src/Context/ThenStepDefinitions.php    |   2 +-
 src/Context/WhenStepDefinitions.php    |   2 +
 tests/tests/TestBehatTags.php          |  25 +++++
 tests/tests/TestMissingCommandHint.php |  56 +++++++++++
 utils/behat-tags.php                   |  99 ++++++++++++++++++
 utils/check-required-tools.php         | 133 +++++++++++++++++++++++++
 10 files changed, 370 insertions(+), 11 deletions(-)
 create mode 100644 tests/tests/TestMissingCommandHint.php
 create mode 100644 utils/check-required-tools.php

diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md
index e67173e91..8822f875e 100644
--- a/.readme-partials/USING.md
+++ b/.readme-partials/USING.md
@@ -350,6 +350,18 @@ they test.
   skipped when there is none, which is the case when the database server runs in a
   container and is only reachable over TCP. Set `WP_CLI_TEST_DBSOCKET` to point at the
   socket if it lives somewhere unusual.
+* `@require-bin-<name>` — the scenario runs a command that is not always installed, such as
+  `@require-bin-svn` or `@require-bin-ex` for one using `ex` as its editor. It is skipped
+  where the command is not available. `@require-composer` does the same for Composer.
+* `@require-extension-<name>` — the scenario needs a PHP extension, such as
+  `@require-extension-imagick`. It is skipped where the extension is not loaded.
+* `@require-download` — the scenario downloads files from WordPress.org. It is skipped where
+  WordPress.org cannot be reached.
+
+Requirement tags the test framework does not know are reported with a warning, as they would
+not filter anything. When a command fails because a command it runs does not exist, the
+failure names the tag to add, and `composer lint-gherkin` lists the scenarios that run
+commands which are not always installed without declaring them.
 
 Before the scenarios run, the scenarios that are left out are listed together with the reason,
 separating the ones the environment cannot run, such as those needing a PHP extension that is
diff --git a/bin/run-gherkin-lint-tests b/bin/run-gherkin-lint-tests
index aa8339695..3a648e0ca 100755
--- a/bin/run-gherkin-lint-tests
+++ b/bin/run-gherkin-lint-tests
@@ -6,16 +6,6 @@ then
 	exit 0;
 fi
 
-# The linter is a Node package, which a PHP project cannot assume is present.
-# Skip rather than fail, the same way the Behat runner skips a package without a
-# behat.yml. The check still runs in CI, where Node is always available.
-if ! command -v npx > /dev/null 2>&1
-then
-	echo 'Did not detect the "npx" command, skipping the Gherkin linting.'
-	echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions."
-	exit 0;
-fi
-
 # To retrieve the WP-CLI tests package root folder, we start with this scripts
 # location.
 SOURCE="$0"
@@ -61,6 +51,21 @@ then
 	set -- features
 fi
 
+# Flag scenarios that run tools which are not always installed without declaring them.
+# This only reports, so that existing feature files can be tagged over time.
+php "${WP_CLI_TESTS_ROOT}/utils/check-required-tools.php" "$@"
+
+# The linter is a Node package, which a PHP project cannot assume is present.
+# Skip rather than fail, the same way the Behat runner skips a package without a
+# behat.yml. The check still runs in CI, where Node is always available.
+if ! command -v npx > /dev/null 2>&1
+then
+	echo 'Did not detect the "npx" command, skipping the Gherkin linting.'
+	echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions."
+	exit 0;
+fi
+
+
 run_linter() {
 	npx --yes "gherkin-lint-plus@${GHERKIN_LINT_VERSION}" --config "${CONFIG_FILE}" "$@"
 }
diff --git a/phpcs.xml.dist b/phpcs.xml.dist
index cad054193..74b4ade1e 100644
--- a/phpcs.xml.dist
+++ b/phpcs.xml.dist
@@ -79,9 +79,11 @@
 		 so this file does not have to comply with WP naming conventions. -->
 	<rule ref="WordPress.NamingConventions.PrefixAllGlobals">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/check-required-tools\.php$</exclude-pattern>
 	</rule>
 	<rule ref="WordPress.WP.GlobalVariablesOverride">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/check-required-tools\.php$</exclude-pattern>
 	</rule>
 
 	<!-- This is a procedural stand-alone file that is adding polyfills when
diff --git a/src/Context/Support.php b/src/Context/Support.php
index 959d3c868..824a496fb 100644
--- a/src/Context/Support.php
+++ b/src/Context/Support.php
@@ -12,6 +12,30 @@
 
 trait Support {
 
+	/**
+	 * Explain how to declare a command that a command run by the scenario needed but could not find.
+	 *
+	 * @param string|null $stderr Error output to inspect. Defaults to the one of the last command run.
+	 * @return string Empty if no command was missing.
+	 */
+	protected function missing_command_hint( $stderr = null ): string {
+		if ( null === $stderr ) {
+			if ( ! isset( $this->result ) ) {
+				return '';
+			}
+			$stderr = (string) $this->result->stderr;
+		}
+
+		if (
+			! preg_match( '/\b(?:sh|bash|dash|zsh)(?:: line \d+|: \d+)?: ([A-Za-z0-9_.+-]+): (?:command )?not found/', $stderr, $matches )
+			&& ! preg_match( "/'([A-Za-z0-9_.+-]+)' is not recognized as an internal or external command/", $stderr, $matches )
+		) {
+			return '';
+		}
+
+		return "\n\nThe command `{$matches[1]}` was not found. If the scenario needs it, tag it with @require-bin-{$matches[1]}, so that environments without it report the scenario as not run instead of failing it.";
+	}
+
 	/**
 	 * @param string $regex
 	 * @param string $actual
@@ -141,6 +165,7 @@ protected function check_string( $output, $expected, $action, $message = false,
 			}
 
 			$message .= "\n\n" . $action_message . "\n" . $expected;
+			$message .= $this->missing_command_hint();
 
 			$diff = $this->generate_diff( $expected, rtrim( $output, "\n" ) );
 			if ( ! empty( $diff ) ) {
diff --git a/src/Context/ThenStepDefinitions.php b/src/Context/ThenStepDefinitions.php
index 2dc700206..28d1d4777 100644
--- a/src/Context/ThenStepDefinitions.php
+++ b/src/Context/ThenStepDefinitions.php
@@ -35,7 +35,7 @@ public function then_the_return_code_should_be( $not, $return_code ): void {
 			( ! $not && (int) $return_code !== $this->result->return_code )
 			|| ( $not && (int) $return_code === $this->result->return_code )
 		) {
-			throw new RuntimeException( $this->result );
+			throw new RuntimeException( $this->result . $this->missing_command_hint() );
 		}
 	}
 
diff --git a/src/Context/WhenStepDefinitions.php b/src/Context/WhenStepDefinitions.php
index e5a6c1237..890a86322 100644
--- a/src/Context/WhenStepDefinitions.php
+++ b/src/Context/WhenStepDefinitions.php
@@ -32,6 +32,8 @@ public function wpcli_tests_invoke_proc( $proc, $mode ) {
 					$message .= "\n\nThe command unexpectedly exited with status {$status}. If you expect a non-zero exit status, use `When I try [...]`. Else, this may be a bug in your code or test.";
 				}
 
+				$message .= $this->missing_command_hint( $e->getMessage() );
+
 				throw new \RuntimeException( $message, $e->getCode(), $e );
 			}
 
diff --git a/tests/tests/TestBehatTags.php b/tests/tests/TestBehatTags.php
index 00e7792e2..a9fb76660 100644
--- a/tests/tests/TestBehatTags.php
+++ b/tests/tests/TestBehatTags.php
@@ -656,4 +656,29 @@ public function test_strict_requirements_allow_other_configurations(): void {
 		$this->assertSame( 0, $result['exit_code'] );
 		$this->assertStringContainsString( '~@broken', $result['stdout'] );
 	}
+
+	public function test_missing_commands_are_filtered_out_and_reported(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'commands.feature',
+			"Feature: Commands\n\n  @require-bin-wpclinonexistentcommand\n  Scenario: Needs a missing command\n    Given a step\n\n  @require-bin-php\n  Scenario: Needs PHP\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams();
+
+		$this->assertStringContainsString( '~@require-bin-wpclinonexistentcommand', $result['stdout'] );
+		$this->assertStringNotContainsString( '@require-bin-php', $result['stdout'] );
+		$this->assertStringContainsString( '@require-bin-wpclinonexistentcommand, as the command "wpclinonexistentcommand" is not available: ', $result['stderr'] );
+	}
+
+	public function test_unknown_requirement_tags_are_warned_about(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'unknown.feature',
+			"Feature: Unknown\n\n  @require-wpclinonexistent\n  Scenario: Has an unknown requirement\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams();
+
+		$this->assertStringContainsString( 'Warning: @require-wpclinonexistent is not a known requirement and does not filter anything: ', $result['stderr'] );
+		$this->assertStringContainsString( 'unknown.feature:4', $result['stderr'] );
+	}
 }
diff --git a/tests/tests/TestMissingCommandHint.php b/tests/tests/TestMissingCommandHint.php
new file mode 100644
index 000000000..c6aa87210
--- /dev/null
+++ b/tests/tests/TestMissingCommandHint.php
@@ -0,0 +1,56 @@
+<?php
+
+namespace WP_CLI\Tests\Tests;
+
+use PHPUnit\Framework\Attributes\DataProvider;
+use WP_CLI\Tests\Context\Support;
+use WP_CLI\Tests\TestCase;
+
+class TestMissingCommandHint extends TestCase {
+
+	/**
+	 * @return string
+	 */
+	private function hint( string $stderr ) {
+		$context = new class() {
+			use Support;
+
+			/**
+			 * @var \WP_CLI\ProcessRun|null
+			 */
+			public $result = null;
+
+			public function hint( string $stderr ): string {
+				return $this->missing_command_hint( $stderr );
+			}
+		};
+
+		return $context->hint( $stderr );
+	}
+
+	/**
+	 * @dataProvider data_missing_commands
+	 */
+	#[DataProvider( 'data_missing_commands' )] // phpcs:ignore PHPCompatibility.Attributes.NewAttributes.PHPUnitAttributeFound
+	public function test_missing_commands_are_named( string $stderr, string $command ): void {
+		$this->assertStringContainsString( "The command `{$command}` was not found. If the scenario needs it, tag it with @require-bin-{$command}", $this->hint( $stderr ) );
+	}
+
+	/**
+	 * @return array<array{string, string}>
+	 */
+	public static function data_missing_commands(): array {
+		return [
+			[ "sh: line 1: ex: command not found\n", 'ex' ],
+			[ "sh: 1: ex: not found\n", 'ex' ],
+			[ "bash: svn: command not found\n", 'svn' ],
+			[ "'zipinfo' is not recognized as an internal or external command,\n", 'zipinfo' ],
+		];
+	}
+
+	public function test_other_errors_get_no_hint(): void {
+		$this->assertSame( '', $this->hint( "Error: not found\n" ) );
+		$this->assertSame( '', $this->hint( "Error: Plugin: not found\n" ) );
+		$this->assertSame( '', $this->hint( '' ) );
+	}
+}
diff --git a/utils/behat-tags.php b/utils/behat-tags.php
index cb8365ba1..f265d7a68 100644
--- a/utils/behat-tags.php
+++ b/utils/behat-tags.php
@@ -384,6 +384,86 @@ function os_tags( $features_folder = 'features' ) {
 	return $skip_tags;
 }
 
+/**
+ * Whether a command is available in the environment.
+ *
+ * @param string $command
+ * @return bool
+ */
+function command_available( $command ) {
+	$check = 'Windows' === PHP_OS_FAMILY ? 'where ' . escapeshellarg( $command ) . ' 2> NUL' : 'command -v ' . escapeshellarg( $command ) . ' 2> /dev/null';
+	exec( $check, $output, $exit_code );
+	return 0 === $exit_code;
+}
+
+/**
+ * Tags for requirements on the environment beyond PHP extensions: commands, network access and opcache settings.
+ *
+ * @param string $features_folder The folder where the feature files are located.
+ * @return array
+ */
+function environment_tags( $features_folder = 'features' ) {
+	$skip_tags = array();
+
+	$feature_files = glob( $features_folder . DIRECTORY_SEPARATOR . '*.feature' );
+	$contents      = '';
+	foreach ( $feature_files ? $feature_files : array() as $feature_file ) {
+		$contents .= (string) file_get_contents( $feature_file ) . "\n";
+	}
+
+	// Commands a scenario needs, such as "@require-bin-svn".
+	if ( preg_match_all( '/@require-bin-([A-Za-z0-9_.+-]+)/', $contents, $matches ) ) {
+		foreach ( array_unique( $matches[1] ) as $command ) {
+			if ( ! command_available( $command ) ) {
+				$skip_tags[] = "@require-bin-{$command}";
+			}
+		}
+	}
+
+	if ( false !== strpos( $contents, '@require-composer' ) && ! command_available( 'composer' ) ) {
+		$skip_tags[] = '@require-composer';
+	}
+
+	if ( false !== strpos( $contents, '@require-download' ) ) {
+		// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- A failed connection is the answer being looked for.
+		$connection = @fsockopen( 'downloads.wordpress.org', 443, $errno, $errstr, 5 );
+		if ( false === $connection ) {
+			$skip_tags[] = '@require-download';
+		} else {
+			fclose( $connection );
+		}
+	}
+
+	// WP-CLI reads the doc comments of commands, which opcache can be configured to drop.
+	if ( false !== strpos( $contents, '@require-opcache-save-comments' ) && extension_loaded( 'Zend OPcache' ) && ! ini_get( 'opcache.save_comments' ) ) {
+		$skip_tags[] = '@require-opcache-save-comments';
+	}
+
+	return $skip_tags;
+}
+
+/**
+ * Warn about requirement tags that the test framework does not know, and which therefore never filter anything.
+ *
+ * @param string $features_folder The folder where the feature files are located.
+ */
+function warn_about_unknown_requirement_tags( $features_folder = 'features' ) {
+	$known = '/^@(require-(wp|php|mysql|mariadb)-[0-9.]+|less-than-(wp|php|mysql|mariadb)-[0-9.]+|require-(mysql|mariadb|sqlite|mysql-or-mariadb|object-cache|wp-stable|mysql-socket|composer|download|opcache-save-comments|windows|macos|linux)|require-(extension|bin)-[A-Za-z0-9_.+-]+)$/';
+
+	$unknown = array();
+	foreach ( scenarios_with_tags( $features_folder ) as $scenario ) {
+		foreach ( $scenario['tags'] as $tag ) {
+			if ( preg_match( '/^@(require|less-than)-/', $tag ) && ! preg_match( $known, $tag ) ) {
+				$unknown[ $tag ][] = $scenario['path'];
+			}
+		}
+	}
+
+	foreach ( $unknown as $tag => $paths ) {
+		fwrite( STDERR, sprintf( "Warning: %s is not a known requirement and does not filter anything: %s\n", $tag, implode( ', ', array_unique( $paths ) ) ) );
+	}
+}
+
 /**
  * Get the scenarios of the feature files with the tags that apply to them.
  *
@@ -448,6 +528,22 @@ function describe_skip_tag( $tag, $context ) {
 		return array( true, "the PHP extension \"{$matches[1]}\" is not loaded" );
 	}
 
+	if ( preg_match( '/^@require-bin-(.+)$/', $tag, $matches ) ) {
+		return array( true, "the command \"{$matches[1]}\" is not available" );
+	}
+
+	if ( '@require-composer' === $tag ) {
+		return array( true, 'the command "composer" is not available' );
+	}
+
+	if ( '@require-download' === $tag ) {
+		return array( true, 'downloads.wordpress.org cannot be reached' );
+	}
+
+	if ( '@require-opcache-save-comments' === $tag ) {
+		return array( true, 'opcache drops doc comments' );
+	}
+
 	if ( '@require-mysql-socket' === $tag ) {
 		return array( true, 'the database server cannot be reached through a socket' );
 	}
@@ -574,6 +670,9 @@ function ( $path ) {
 
 $skip_tags = array_merge( $skip_tags, extension_tags( $features_folder ) );
 $skip_tags = array_merge( $skip_tags, os_tags( $features_folder ) );
+$skip_tags = array_merge( $skip_tags, environment_tags( $features_folder ) );
+
+warn_about_unknown_requirement_tags( $features_folder );
 
 $missing = report_filtered_scenarios(
 	$skip_tags,
diff --git a/utils/check-required-tools.php b/utils/check-required-tools.php
new file mode 100644
index 000000000..84b4ecd17
--- /dev/null
+++ b/utils/check-required-tools.php
@@ -0,0 +1,133 @@
+<?php
+/**
+ * Flag scenarios that run external tools without declaring them.
+ *
+ * A scenario that runs a tool that is not always installed has to carry the matching
+ * `@require-bin-<name>` tag, so that environments without the tool report the scenario
+ * as not run instead of failing it halfway through.
+ *
+ * This only reports, so that existing feature files can be tagged over time.
+ *
+ * Usage: php check-required-tools.php [<features folder or file>...]
+ */
+
+/**
+ * Tools that are not part of every environment the tests run in.
+ *
+ * Common ones like git, curl or the database client are assumed to be present.
+ */
+const WP_CLI_TESTS_OPTIONAL_TOOLS = [ 'convert', 'ex', 'gs', 'identify', 'jq', 'nano', 'phpunit', 'sqlite3', 'svn', 'tar', 'unzip', 'vi', 'vim', 'zip', 'zipinfo' ];
+
+/**
+ * Get the tools that a command line runs.
+ *
+ * @param string $command_line
+ * @return array<string>
+ */
+function wp_cli_tests_tools_in_command( $command_line ) {
+	$tools = [];
+
+	foreach ( preg_split( '/\|\||&&|;|\||\$\(/', $command_line ) as $part ) {
+		// Commands can be set as editors through environment variables.
+		if ( preg_match_all( '/\b(?:EDITOR|VISUAL)=["\']?([A-Za-z0-9_.+-]+)/', $part, $matches ) ) {
+			$tools = array_merge( $tools, $matches[1] );
+		}
+
+		$part = (string) preg_replace( '/^\s*(?:[A-Z_][A-Z0-9_]*=(?:"[^"]*"|\'[^\']*\'|\S+)\s+)*/', '', $part );
+		if ( preg_match( '/^([A-Za-z0-9_.+-]+)/', $part, $matches ) ) {
+			$tools[] = $matches[1];
+		}
+	}
+
+	return array_values( array_intersect( array_unique( $tools ), WP_CLI_TESTS_OPTIONAL_TOOLS ) );
+}
+
+/**
+ * Check one feature file.
+ *
+ * @param string $file
+ * @return array<string> Problems found, one per scenario and tool.
+ */
+function wp_cli_tests_check_feature( $file ) {
+	$problems     = [];
+	$feature_tags = [];
+	$pending_tags = [];
+	$background   = [];
+	$current      = null;
+	$doc_string   = null;
+	$scenarios    = [];
+
+	foreach ( (array) file( $file, FILE_IGNORE_NEW_LINES ) as $index => $line ) {
+		$line = trim( (string) $line );
+
+		if ( null !== $doc_string ) {
+			if ( 0 === strpos( $line, $doc_string ) ) {
+				$doc_string = null;
+			}
+			continue;
+		}
+
+		if ( 0 === strpos( $line, '"""' ) || 0 === strpos( $line, '```' ) ) {
+			$doc_string = substr( $line, 0, 3 );
+			continue;
+		}
+
+		if ( 0 === strpos( $line, '@' ) ) {
+			preg_match_all( '/@[^\s@]+/', $line, $matches );
+			$pending_tags = array_merge( $pending_tags, $matches[0] );
+		} elseif ( preg_match( '/^Feature:/', $line ) ) {
+			$feature_tags = $pending_tags;
+			$pending_tags = [];
+		} elseif ( preg_match( '/^Background:/', $line ) ) {
+			$current = 'background';
+		} elseif ( preg_match( '/^(Scenario|Scenario Outline|Scenario Template|Example):/', $line ) ) {
+			$scenarios[]  = [
+				'line'  => $index + 1,
+				'tags'  => array_merge( $feature_tags, $pending_tags ),
+				'tools' => [],
+			];
+			$current      = count( $scenarios ) - 1;
+			$pending_tags = [];
+		} elseif ( preg_match( '/^(?:Given|When|Then|And|But) I (?:run|try|launch in the background) `([^`]+)`/', $line, $matches ) ) {
+			$tools = wp_cli_tests_tools_in_command( $matches[1] );
+			if ( 'background' === $current ) {
+				$background = array_merge( $background, $tools );
+			} elseif ( null !== $current ) {
+				$scenarios[ $current ]['tools'] = array_merge( $scenarios[ $current ]['tools'], $tools );
+			}
+		}
+	}
+
+	foreach ( $scenarios as $scenario ) {
+		foreach ( array_unique( array_merge( $background, $scenario['tools'] ) ) as $tool ) {
+			if ( ! in_array( "@require-bin-{$tool}", $scenario['tags'], true ) ) {
+				$problems[] = "{$file}:{$scenario['line']} runs `{$tool}` without the @require-bin-{$tool} tag.";
+			}
+		}
+	}
+
+	return $problems;
+}
+
+$paths = array_slice( $argv, 1 );
+if ( ! $paths ) {
+	$paths = [ 'features' ];
+}
+
+$files = [];
+foreach ( $paths as $path ) {
+	if ( is_dir( $path ) ) {
+		$files = array_merge( $files, (array) glob( rtrim( $path, '/\\' ) . '/*.feature' ) );
+	} elseif ( is_file( $path ) ) {
+		$files[] = $path;
+	}
+}
+
+$problems = [];
+foreach ( $files as $file ) {
+	$problems = array_merge( $problems, wp_cli_tests_check_feature( (string) $file ) );
+}
+
+foreach ( $problems as $problem ) {
+	echo "Warning: {$problem}\n";
+}