diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index d9a4208b2..e06e373d4 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -32,7 +32,8 @@ true, \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE, true, - \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE + \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE, + false ); // Load needed framework env params diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index 097150888..cadc1c6a8 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -12,6 +12,7 @@ use Magento\FunctionalTestingFramework\Test\Objects\TestObject; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; class TestGeneratorTest extends MagentoTestCase { @@ -38,4 +39,51 @@ public function testEntityException() $testGeneratorObject->createAllTestFiles(null, []); } + + /** + * Tests that skipped tests do not have a fully generated body + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testSkippedNoGeneration() + { + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertContains('This test is skipped', $output); + $this->assertNotContains($actionInput, $output); + } + + /** + * Tests that skipped tests have a fully generated body when --allowSkipped is passed in + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testAllowSkipped() + { + // Mock allowSkipped for TestGenerator + AspectMock::double(MftfApplicationConfig::class, ['allowSkipped' => true]); + + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertNotContains('This test is skipped', $output); + $this->assertContains($actionInput, $output); + } } diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 2301d6f44..aea53c96f 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -122,6 +122,7 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove] | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| +| `--allow-skipped` | Allows MFTF to generate and run tests marked with `.`| | `--debug or --debug=[]`| Performs schema validations on XML files.
DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered.
DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.

NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.
| | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.| diff --git a/etc/config/.env.example b/etc/config/.env.example index 369fd946b..f4aa90e51 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -49,6 +49,9 @@ MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_C #*** Bool property which allows the user to toggle debug output during test execution #MFTF_DEBUG= +#*** Bool property which allows the user to generate and run tests marked as skipped +#ALLOW_SKIPPED=true + #*** Default timeout for wait actions #WAIT_TIMEOUT=10 #*** End of .env ***# diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 29278f761..6faf322bc 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -9,6 +9,9 @@ class MftfApplicationConfig { + /** + * MFTF Execution Phases + */ const GENERATION_PHASE = "generation"; const EXECUTION_PHASE = "execution"; const UNIT_TEST_PHASE = "testing"; @@ -50,6 +53,12 @@ class MftfApplicationConfig */ private $debugLevel; + /** + * Boolean which allows MFTF to fully generate skipped tests + * @var boolean + */ + private $allowSkipped; + /** * MftfApplicationConfig Singelton Instance * @@ -64,13 +73,15 @@ class MftfApplicationConfig * @param string $phase * @param boolean $verboseEnabled * @param string $debugLevel + * @param boolean $allowSkipped * @throws TestFrameworkException */ private function __construct( $forceGenerate = false, $phase = self::EXECUTION_PHASE, $verboseEnabled = null, - $debugLevel = self::LEVEL_NONE + $debugLevel = self::LEVEL_NONE, + $allowSkipped = false ) { $this->forceGenerate = $forceGenerate; @@ -89,6 +100,7 @@ private function __construct( default: $this->debugLevel = self::LEVEL_DEVELOPER; } + $this->allowSkipped = $allowSkipped; } /** @@ -99,13 +111,15 @@ private function __construct( * @param string $phase * @param boolean $verboseEnabled * @param string $debugLevel + * @param boolean $allowSkipped * @return void + * @throws TestFrameworkException */ - public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel) + public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = - new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel); + new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped); } } @@ -113,6 +127,7 @@ public static function create($forceGenerate, $phase, $verboseEnabled, $debugLev * This function returns an instance of the MftfApplicationConfig which is created once the application starts. * * @return MftfApplicationConfig + * @throws TestFrameworkException */ public static function getConfig() { @@ -157,6 +172,16 @@ public function getDebugLevel() return $this->debugLevel ?? getenv('MFTF_DEBUG'); } + /** + * Returns a boolean indicating whether mftf is generating skipped tests. + * + * @return boolean + */ + public function allowSkipped() + { + return $this->allowSkipped ?? getenv('ALLOW_SKIPPED'); + } + /** * Returns a string which indicates the phase of mftf execution. * diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index d4044c74b..75c521379 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; class BaseGenerateCommand extends Command { @@ -28,6 +29,23 @@ protected function configure() 'r', InputOption::VALUE_NONE, 'remove previous generated suites and tests' + )->addOption( + "force", + 'f', + InputOption::VALUE_NONE, + 'force generation and running of tests regardless of Magento Instance Configuration' + )->addOption( + "allowSkipped", + 'a', + InputOption::VALUE_NONE, + 'Allows MFTF to generate and run skipped tests.' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when generating and running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 23651b492..581688615 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -36,11 +36,6 @@ protected function configure() 'name(s) of specific tests to generate' )->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default') ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'Force generation of tests regardless of Magento Instance Configuration' - )->addOption( 'time', 'i', InputOption::VALUE_REQUIRED, @@ -51,13 +46,6 @@ protected function configure() 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration' - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -83,6 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); + $allowSkipped = $input->getOption('allowSkipped'); if ($json !== null && !json_decode($json)) { // stop execution if we have failed to properly parse any json passed in by the user @@ -100,7 +89,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ($debug !== MftfApplicationConfig::LEVEL_NONE)); } - $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose); + $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped); // create our manifest file here $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); @@ -128,18 +117,26 @@ protected function execute(InputInterface $input, OutputInterface $output) * @param boolean $force * @param string $debug * @param boolean $verbose + * @param boolean $allowSkipped * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose) - { + private function createTestConfiguration( + string $json, + array $tests, + bool $force, + string $debug, + bool $verbose, + bool $allowSkipped + ) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( $force, MftfApplicationConfig::GENERATION_PHASE, $verbose, - $debug + $debug, + $allowSkipped ); $testConfiguration = []; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 76a8bc8f4..fab65a8e6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -31,20 +31,7 @@ protected function configure() 'name', InputArgument::REQUIRED | InputArgument::IS_ARRAY, "name of tests to generate and execute" - )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test") - ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test"); parent::configure(); } @@ -66,6 +53,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -83,7 +71,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]), '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index d6a580d52..ed00322ff 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -50,15 +50,7 @@ class RunTestFailedCommand extends BaseGenerateCommand protected function configure() { $this->setName('run:failed') - ->setDescription('Execute a set of tests referenced via failed file') - ->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + ->setDescription('Execute a set of tests referenced via failed file'); parent::configure(); } @@ -76,13 +68,17 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); + // Create Mftf Configuration MftfApplicationConfig::create( - false, + $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); $testConfiguration = $this->getFailedTestList(); @@ -96,9 +92,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $args = [ '--tests' => $testConfiguration, '--remove' => true, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; - $command->run(new ArrayInput($args), $output); $testManifestList = $this->readTestManifestFile(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index d714a1dd1..81a6b864f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -34,18 +34,6 @@ protected function configure() 'k', InputOption::VALUE_NONE, "only execute a group of tests without generating from source xml" - )->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT )->addArgument( 'groups', InputArgument::IS_ARRAY | InputArgument::REQUIRED, @@ -72,6 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -85,7 +74,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); if (!$skipGeneration) { @@ -95,7 +85,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--tests' => $testConfiguration, '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index a9eb15152..1facfe746 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Util; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; @@ -227,14 +228,14 @@ public function createAllTestFiles($testManifest = null, $testsToIgnore = null) * @throws TestReferenceException * @throws \Exception */ - private function assembleTestPhp($testObject) + public function assembleTestPhp($testObject) { $usePhp = $this->generateUseStatementsPhp(); $classAnnotationsPhp = $this->generateAnnotationsPhp($testObject->getAnnotations()); $className = $testObject->getCodeceptionName(); try { - if (!$testObject->isSkipped()) { + if (!$testObject->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $hookPhp = $this->generateHooksPhp($testObject->getHooks()); } else { $hookPhp = null; @@ -1606,7 +1607,7 @@ private function generateTestPhp($test) $testName = str_replace(' ', '', $testName); $testAnnotations = $this->generateAnnotationsPhp($test->getAnnotations(), true); $dependencies = 'AcceptanceTester $I'; - if ($test->isSkipped()) { + if ($test->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $skipString = "This test is skipped due to the following issues:\\n"; $issues = $test->getAnnotations()['skip'] ?? null; if (isset($issues)) {