Declare and check the commands that scenarios need

.readme-partials/USING.md

@@ -350,6 +350,18 @@ they test.
   skipped when there is none, which is the case when the database server runs in a
   container and is only reachable over TCP. Set `WP_CLI_TEST_DBSOCKET` to point at the
   socket if it lives somewhere unusual.
+* `@require-bin-<name>` — the scenario runs a command that is not always installed, such as
+  `@require-bin-svn` or `@require-bin-ex` for one using `ex` as its editor. It is skipped
+  where the command is not available. `@require-composer` does the same for Composer.
+* `@require-extension-<name>` — the scenario needs a PHP extension, such as
+  `@require-extension-imagick`. It is skipped where the extension is not loaded.
+* `@require-download` — the scenario downloads files from WordPress.org. It is skipped where
+  WordPress.org cannot be reached.
+
+Requirement tags the test framework does not know are reported with a warning, as they would
+not filter anything. When a command fails because a command it runs does not exist, the
+failure names the tag to add, and `composer lint-gherkin` lists the scenarios that run
+commands which are not always installed without declaring them.
 
 Before the scenarios run, the scenarios that are left out are listed together with the reason,
 separating the ones the environment cannot run, such as those needing a PHP extension that is

bin/run-gherkin-lint-tests

@@ -6,16 +6,6 @@ then
 	exit 0;
 fi
 
-# The linter is a Node package, which a PHP project cannot assume is present.
-# Skip rather than fail, the same way the Behat runner skips a package without a
-# behat.yml. The check still runs in CI, where Node is always available.
-if ! command -v npx > /dev/null 2>&1
-then
-	echo 'Did not detect the "npx" command, skipping the Gherkin linting.'
-	echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions."
-	exit 0;
-fi
-
 # To retrieve the WP-CLI tests package root folder, we start with this scripts
 # location.
 SOURCE="$0"
@@ -61,6 +51,21 @@ then
 	set -- features
 fi
 
+# Flag scenarios that run tools which are not always installed without declaring them.
+# This only reports, so that existing feature files can be tagged over time.
+php "${WP_CLI_TESTS_ROOT}/utils/check-required-tools.php" "$@"
+
+# The linter is a Node package, which a PHP project cannot assume is present.
+# Skip rather than fail, the same way the Behat runner skips a package without a
+# behat.yml. The check still runs in CI, where Node is always available.
+if ! command -v npx > /dev/null 2>&1
+then
+	echo 'Did not detect the "npx" command, skipping the Gherkin linting.'
+	echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions."
+	exit 0;
+fi
+
+
 run_linter() {
 	npx --yes "gherkin-lint-plus@${GHERKIN_LINT_VERSION}" --config "${CONFIG_FILE}" "$@"
 }

phpcs.xml.dist

@@ -79,9 +79,11 @@
 		 so this file does not have to comply with WP naming conventions. -->
 	<rule ref="WordPress.NamingConventions.PrefixAllGlobals">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/check-required-tools\.php$</exclude-pattern>
 	</rule>
 	<rule ref="WordPress.WP.GlobalVariablesOverride">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/check-required-tools\.php$</exclude-pattern>
 	</rule>
 
 	<!-- This is a procedural stand-alone file that is adding polyfills when

src/Context/Support.php

@@ -12,6 +12,30 @@
 
 trait Support {
 
+	/**
+	 * Explain how to declare a command that a command run by the scenario needed but could not find.
+	 *
+	 * @param string|null $stderr Error output to inspect. Defaults to the one of the last command run.
+	 * @return string Empty if no command was missing.
+	 */
+	protected function missing_command_hint( $stderr = null ): string {
+		if ( null === $stderr ) {
+			if ( ! isset( $this->result ) ) {
+				return '';
+			}
+			$stderr = (string) $this->result->stderr;
+		}
+
+		if (
+			! preg_match( '/\b(?:sh|bash|dash|zsh)(?:: line \d+|: \d+)?: ([A-Za-z0-9_.+-]+): (?:command )?not found/', $stderr, $matches )
+			&& ! preg_match( "/'([A-Za-z0-9_.+-]+)' is not recognized as an internal or external command/", $stderr, $matches )
+		) {
+			return '';
+		}
+
+		return "\n\nThe command `{$matches[1]}` was not found. If the scenario needs it, tag it with @require-bin-{$matches[1]}, so that environments without it report the scenario as not run instead of failing it.";
+	}
+
 	/**
 	 * @param string $regex
 	 * @param string $actual
@@ -141,6 +165,7 @@ protected function check_string( $output, $expected, $action, $message = false,
 			}
 
 			$message .= "\n\n" . $action_message . "\n" . $expected;
+			$message .= $this->missing_command_hint();
 
 			$diff = $this->generate_diff( $expected, rtrim( $output, "\n" ) );
 			if ( ! empty( $diff ) ) {

src/Context/ThenStepDefinitions.php

@@ -35,7 +35,7 @@ public function then_the_return_code_should_be( $not, $return_code ): void {
 			( ! $not && (int) $return_code !== $this->result->return_code )
 			|| ( $not && (int) $return_code === $this->result->return_code )
 		) {
-			throw new RuntimeException( $this->result );
+			throw new RuntimeException( $this->result . $this->missing_command_hint() );
 		}
 	}
 

src/Context/WhenStepDefinitions.php

@@ -32,6 +32,8 @@ public function wpcli_tests_invoke_proc( $proc, $mode ) {
 					$message .= "\n\nThe command unexpectedly exited with status {$status}. If you expect a non-zero exit status, use `When I try [...]`. Else, this may be a bug in your code or test.";
 				}
 
+				$message .= $this->missing_command_hint( $e->getMessage() );
+
 				throw new \RuntimeException( $message, $e->getCode(), $e );
 			}
 

tests/tests/TestBehatTags.php

@@ -656,4 +656,29 @@ public function test_strict_requirements_allow_other_configurations(): void {
 		$this->assertSame( 0, $result['exit_code'] );
 		$this->assertStringContainsString( '~@broken', $result['stdout'] );
 	}
+
+	public function test_missing_commands_are_filtered_out_and_reported(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'commands.feature',
+			"Feature: Commands\n\n  @require-bin-wpclinonexistentcommand\n  Scenario: Needs a missing command\n    Given a step\n\n  @require-bin-php\n  Scenario: Needs PHP\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams();
+
+		$this->assertStringContainsString( '~@require-bin-wpclinonexistentcommand', $result['stdout'] );
+		$this->assertStringNotContainsString( '@require-bin-php', $result['stdout'] );
+		$this->assertStringContainsString( '@require-bin-wpclinonexistentcommand, as the command "wpclinonexistentcommand" is not available: ', $result['stderr'] );
+	}
+
+	public function test_unknown_requirement_tags_are_warned_about(): void {
+		file_put_contents(
+			$this->temp_dir . DIRECTORY_SEPARATOR . 'features' . DIRECTORY_SEPARATOR . 'unknown.feature',
+			"Feature: Unknown\n\n  @require-wpclinonexistent\n  Scenario: Has an unknown requirement\n    Given a step\n"
+		);
+
+		$result = $this->run_behat_tags_script_with_streams();
+
+		$this->assertStringContainsString( 'Warning: @require-wpclinonexistent is not a known requirement and does not filter anything: ', $result['stderr'] );
+		$this->assertStringContainsString( 'unknown.feature:4', $result['stderr'] );
+	}
 }

tests/tests/TestMissingCommandHint.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace WP_CLI\Tests\Tests;
+
+use PHPUnit\Framework\Attributes\DataProvider;
+use WP_CLI\Tests\Context\Support;
+use WP_CLI\Tests\TestCase;
+
+class TestMissingCommandHint extends TestCase {
+
+	/**
+	 * @return string
+	 */
+	private function hint( string $stderr ) {
+		$context = new class() {
+			use Support;
+
+			/**
+			 * @var \WP_CLI\ProcessRun|null
+			 */
+			public $result = null;
+
+			public function hint( string $stderr ): string {
+				return $this->missing_command_hint( $stderr );
+			}
+		};
+
+		return $context->hint( $stderr );
+	}
+
+	/**
+	 * @dataProvider data_missing_commands
+	 */
+	#[DataProvider( 'data_missing_commands' )] // phpcs:ignore PHPCompatibility.Attributes.NewAttributes.PHPUnitAttributeFound
+	public function test_missing_commands_are_named( string $stderr, string $command ): void {
+		$this->assertStringContainsString( "The command `{$command}` was not found. If the scenario needs it, tag it with @require-bin-{$command}", $this->hint( $stderr ) );
+	}
+
+	/**
+	 * @return array<array{string, string}>
+	 */
+	public static function data_missing_commands(): array {
+		return [
+			[ "sh: line 1: ex: command not found\n", 'ex' ],
+			[ "sh: 1: ex: not found\n", 'ex' ],
+			[ "bash: svn: command not found\n", 'svn' ],
+			[ "'zipinfo' is not recognized as an internal or external command,\n", 'zipinfo' ],
+		];
+	}
+
+	public function test_other_errors_get_no_hint(): void {
+		$this->assertSame( '', $this->hint( "Error: not found\n" ) );
+		$this->assertSame( '', $this->hint( "Error: Plugin: not found\n" ) );
+		$this->assertSame( '', $this->hint( '' ) );
+	}
+}

utils/behat-tags.php

@@ -384,6 +384,86 @@ function os_tags( $features_folder = 'features' ) {
 	return $skip_tags;
 }
 
+/**
+ * Whether a command is available in the environment.
+ *
+ * @param string $command
+ * @return bool
+ */
+function command_available( $command ) {
+	$check = 'Windows' === PHP_OS_FAMILY ? 'where ' . escapeshellarg( $command ) . ' 2> NUL' : 'command -v ' . escapeshellarg( $command ) . ' 2> /dev/null';
+	exec( $check, $output, $exit_code );
+	return 0 === $exit_code;
+}
+
+/**
+ * Tags for requirements on the environment beyond PHP extensions: commands, network access and opcache settings.
+ *
+ * @param string $features_folder The folder where the feature files are located.
+ * @return array
+ */
+function environment_tags( $features_folder = 'features' ) {
+	$skip_tags = array();
+
+	$feature_files = glob( $features_folder . DIRECTORY_SEPARATOR . '*.feature' );
+	$contents      = '';
+	foreach ( $feature_files ? $feature_files : array() as $feature_file ) {
+		$contents .= (string) file_get_contents( $feature_file ) . "\n";
+	}
+
+	// Commands a scenario needs, such as "@require-bin-svn".
+	if ( preg_match_all( '/@require-bin-([A-Za-z0-9_.+-]+)/', $contents, $matches ) ) {
+		foreach ( array_unique( $matches[1] ) as $command ) {
+			if ( ! command_available( $command ) ) {
+				$skip_tags[] = "@require-bin-{$command}";
+			}
+		}
+	}
+
+	if ( false !== strpos( $contents, '@require-composer' ) && ! command_available( 'composer' ) ) {
+		$skip_tags[] = '@require-composer';
+	}
+
+	if ( false !== strpos( $contents, '@require-download' ) ) {
+		// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- A failed connection is the answer being looked for.
+		$connection = @fsockopen( 'downloads.wordpress.org', 443, $errno, $errstr, 5 );
+		if ( false === $connection ) {
+			$skip_tags[] = '@require-download';
+		} else {
+			fclose( $connection );
+		}
+	}
+
+	// WP-CLI reads the doc comments of commands, which opcache can be configured to drop.
+	if ( false !== strpos( $contents, '@require-opcache-save-comments' ) && extension_loaded( 'Zend OPcache' ) && ! ini_get( 'opcache.save_comments' ) ) {
+		$skip_tags[] = '@require-opcache-save-comments';
+	}
+
+	return $skip_tags;
+}
+
+/**
+ * Warn about requirement tags that the test framework does not know, and which therefore never filter anything.
+ *
+ * @param string $features_folder The folder where the feature files are located.
+ */
+function warn_about_unknown_requirement_tags( $features_folder = 'features' ) {
+	$known = '/^@(require-(wp|php|mysql|mariadb)-[0-9.]+|less-than-(wp|php|mysql|mariadb)-[0-9.]+|require-(mysql|mariadb|sqlite|mysql-or-mariadb|object-cache|wp-stable|mysql-socket|composer|download|opcache-save-comments|windows|macos|linux)|require-(extension|bin)-[A-Za-z0-9_.+-]+)$/';
+
+	$unknown = array();
+	foreach ( scenarios_with_tags( $features_folder ) as $scenario ) {
+		foreach ( $scenario['tags'] as $tag ) {
+			if ( preg_match( '/^@(require|less-than)-/', $tag ) && ! preg_match( $known, $tag ) ) {
+				$unknown[ $tag ][] = $scenario['path'];
+			}
+		}
+	}
+
+	foreach ( $unknown as $tag => $paths ) {
+		fwrite( STDERR, sprintf( "Warning: %s is not a known requirement and does not filter anything: %s\n", $tag, implode( ', ', array_unique( $paths ) ) ) );
+	}
+}
+
 /**
  * Get the scenarios of the feature files with the tags that apply to them.
  *
@@ -448,6 +528,22 @@ function describe_skip_tag( $tag, $context ) {
 		return array( true, "the PHP extension \"{$matches[1]}\" is not loaded" );
 	}
 
+	if ( preg_match( '/^@require-bin-(.+)$/', $tag, $matches ) ) {
+		return array( true, "the command \"{$matches[1]}\" is not available" );
+	}
+
+	if ( '@require-composer' === $tag ) {
+		return array( true, 'the command "composer" is not available' );
+	}
+
+	if ( '@require-download' === $tag ) {
+		return array( true, 'downloads.wordpress.org cannot be reached' );
+	}
+
+	if ( '@require-opcache-save-comments' === $tag ) {
+		return array( true, 'opcache drops doc comments' );
+	}
+
 	if ( '@require-mysql-socket' === $tag ) {
 		return array( true, 'the database server cannot be reached through a socket' );
 	}
@@ -574,6 +670,9 @@ function ( $path ) {
 
 $skip_tags = array_merge( $skip_tags, extension_tags( $features_folder ) );
 $skip_tags = array_merge( $skip_tags, os_tags( $features_folder ) );
+$skip_tags = array_merge( $skip_tags, environment_tags( $features_folder ) );
+
+warn_about_unknown_requirement_tags( $features_folder );
 
 $missing = report_filtered_scenarios(
 	$skip_tags,