Run scenarios in parallel workers

.readme-partials/USING.md

@@ -254,6 +254,17 @@ composer behat -- --stop-on-failure
 composer behat-rerun
 ```
 
+### Running scenarios in parallel
+
+Set `WP_CLI_TEST_PARALLEL` to a number of workers to spread the scenarios over that many Behat processes:
+```bash
+WP_CLI_TEST_PARALLEL=4 composer behat
+```
+
+Every worker but the first gets its own temporary directory, and with it its own home directory, caches and run directories, as well as its own test database, named after the test database with a numeric suffix (`wp_cli_test_1`, `wp_cli_test_2`, ...). Run `composer prepare-tests` once to allow the test user to create these databases. Without that permission, the scenarios are run one after the other, as they are on Windows, when collecting code coverage, or when selecting scenarios by path or name.
+
+Scenarios that rely on resources shared across the system, such as a fixed port or a fixed path in the system's temporary directory, are all run by the first worker, one after the other. `composer behat-rerun` re-runs exactly the scenarios that failed in the last parallel run.
+
 ### Linting the feature files
 
 `composer lint-gherkin` checks `features/` with

bin/install-package-tests

@@ -150,6 +150,8 @@ install_mysql_db_8_0_plus() {
 	${CLIENT_BINARY} -e "CREATE USER IF NOT EXISTS \`${TEST_USER}\`@'%' IDENTIFIED WITH caching_sha2_password BY '${TEST_PASSWORD}'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}_scaffold\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
+	# The databases of parallel test workers, named after the test database with a numeric suffix.
+	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}\\_%\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	{ set +ex; } 2> /dev/null # stop printing the commands
 }
 
@@ -159,6 +161,8 @@ install_mysql_db_lower_than_8_0() {
 	${CLIENT_BINARY} -e "CREATE DATABASE IF NOT EXISTS \`${TEST_DB}\`;" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL ON \`${TEST_DB}\`.* TO '${TEST_USER}'@'%' IDENTIFIED BY '${TEST_PASSWORD}'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL ON \`${TEST_DB}_scaffold\`.* TO '${TEST_USER}'@'%' IDENTIFIED BY '${TEST_PASSWORD}'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
+	# The databases of parallel test workers, named after the test database with a numeric suffix.
+	${CLIENT_BINARY} -e "GRANT ALL ON \`${TEST_DB}\\_%\`.* TO '${TEST_USER}'@'%' IDENTIFIED BY '${TEST_PASSWORD}'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	{ set +ex; } 2> /dev/null # stop printing the commands
 }
 
@@ -169,6 +173,8 @@ install_mariadb() {
 	${CLIENT_BINARY} -e "CREATE USER IF NOT EXISTS \`${TEST_USER}\`@'%' IDENTIFIED BY '${TEST_PASSWORD}'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}_scaffold\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
+	# The databases of parallel test workers, named after the test database with a numeric suffix.
+	${CLIENT_BINARY} -e "GRANT ALL PRIVILEGES ON \`${TEST_DB}\\_%\`.* TO '${TEST_USER}'@'%'" ${HOST_STRING} -u"${USER}" "${PASSWORD_STRING}"
 }
 
 if [ "${TYPE}" = "MariaDB" ]; then

bin/run-behat-tests

@@ -78,9 +78,10 @@ else
 
 		DBNAME=${WP_CLI_TEST_DBNAME:-wp_cli_test}
 
-		# We need to test the connection.
-		# Let's try to connect to the specific test database.
-		if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="USE \`${DBNAME}\`;" 2>/dev/null; then
+		# We need to test the connection, and that the test user may create the test database.
+		# The database itself does not need to exist, as the scenarios drop and create it, which
+		# is also why a rerun after a run in parallel workers usually finds it missing.
+		if ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}\`;" 2>/dev/null; then
 			echo "Warning: Could not connect to the MySQL/MariaDB database."
 			echo "Please make sure the database is running and run 'composer prepare-tests' once to set it up."
 			echo "The tests will continue to be run, but with WP_CLI_TEST_DBTYPE=sqlite."
@@ -270,4 +271,39 @@ for arg in "$@"; do
 	fi
 done
 
+# Run the scenarios in parallel workers when WP_CLI_TEST_PARALLEL is set to their number.
+# Every worker but the first uses its own test database, which the test user needs
+# to be allowed to create, as set up by "composer prepare-tests".
+PARALLEL="${WP_CLI_TEST_PARALLEL-}"
+
+# A rerun after a parallel run covers exactly the scenarios that failed in it, which only the
+# parallel runner knows, whether or not WP_CLI_TEST_PARALLEL is set for the rerun as well.
+for arg in "$@"; do
+	if [ "$arg" = "--rerun" ] && [ -f "${TMPDIR:-/tmp}/wp-cli-test-parallel-failures.scenarios" ]; then
+		exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php 1 --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@"
+	fi
+done
+
+case "$(uname -s)" in
+	MINGW*|MSYS*|CYGWIN*)
+		PARALLEL="";;
+esac
+if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then
+	if ! is_numeric "${PARALLEL}"; then
+		echo "Warning: WP_CLI_TEST_PARALLEL is not a number of workers, running the scenarios one after the other." >&2
+		PARALLEL=""
+	elif [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]]; then
+		echo "Code coverage is collected, running the scenarios one after the other." >&2
+		PARALLEL=""
+	elif [ "${WP_CLI_TEST_DBTYPE-}" != "sqlite" ] && ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}_1\`; DROP DATABASE \`${DBNAME}_1\`;" 2>/dev/null; then
+		echo "Warning: The test user cannot create the databases of parallel workers, running the scenarios one after the other." >&2
+		echo "Run 'composer prepare-tests' once to allow it." >&2
+		PARALLEL=""
+	fi
+fi
+
+if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then
+	exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@"
+fi
+
 vendor/bin/behat --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" "$@"

phpcs.xml.dist

@@ -79,9 +79,11 @@
 		 so this file does not have to comply with WP naming conventions. -->
 	<rule ref="WordPress.NamingConventions.PrefixAllGlobals">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/behat-parallel\.php$</exclude-pattern>
 	</rule>
 	<rule ref="WordPress.WP.GlobalVariablesOverride">
 		<exclude-pattern>*/utils/behat-tags\.php$</exclude-pattern>
+		<exclude-pattern>*/utils/behat-parallel\.php$</exclude-pattern>
 	</rule>
 
 	<!-- This is a procedural stand-alone file that is adding polyfills when

src/Context/FeatureContext.php

@@ -481,7 +481,9 @@ private static function get_process_env_variables(): array {
 
 		$env = array_merge( $_ENV, $env );
 
-		foreach ( [ 'TEMP', 'TMP', 'SystemRoot', 'windir' ] as $key ) {
+		// Keep the launched processes in the same temporary directory as the tests, which differs per
+		// parallel worker. WordPress saves downloads under fixed names there, which would clash otherwise.
+		foreach ( [ 'TMPDIR', 'TEMP', 'TMP', 'SystemRoot', 'windir' ] as $key ) {
 			$value = getenv( $key );
 			if ( false !== $value ) {
 				$env[ $key ] = $value;