From b27e2a9b6ff8602551ca8535912a942022d1ebf7 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 08:47:56 -0500 Subject: [PATCH 1/7] MQE-1582: Enable Testers To Run Skipped Tests - added --allowSkipped flag, added to base generate command - moved --force flag to base generate command - fixed some existing issues around not all BaseGenerate inheriting commands not making proper use of --force flag --- .../Config/MftfApplicationConfig.php | 35 ++++++++++++++++--- .../Console/BaseGenerateCommand.php | 10 ++++++ .../Console/GenerateTestsCommand.php | 14 ++++---- .../Console/RunTestCommand.php | 10 +++--- .../Console/RunTestFailedCommand.php | 12 ++++--- .../Console/RunTestGroupCommand.php | 12 +++---- .../Util/TestGenerator.php | 3 +- 7 files changed, 65 insertions(+), 31 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 29278f761..799798640 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -9,6 +9,9 @@ class MftfApplicationConfig { + /** + * MFTF Execution Phases + */ const GENERATION_PHASE = "generation"; const EXECUTION_PHASE = "execution"; const UNIT_TEST_PHASE = "testing"; @@ -50,6 +53,12 @@ class MftfApplicationConfig */ private $debugLevel; + /** + * Boolean which allows MFTF to fully generate skipped tests + * @var bool + */ + private $allowSkipped; + /** * MftfApplicationConfig Singelton Instance * @@ -64,13 +73,15 @@ class MftfApplicationConfig * @param string $phase * @param boolean $verboseEnabled * @param string $debugLevel + * @param boolean $allowSkipped * @throws TestFrameworkException */ private function __construct( $forceGenerate = false, $phase = self::EXECUTION_PHASE, $verboseEnabled = null, - $debugLevel = self::LEVEL_NONE + $debugLevel = self::LEVEL_NONE, + $allowSkipped = false ) { $this->forceGenerate = $forceGenerate; @@ -89,6 +100,7 @@ private function __construct( default: $this->debugLevel = self::LEVEL_DEVELOPER; } + $this->allowSkipped = $allowSkipped; } /** @@ -96,16 +108,18 @@ private function __construct( * returns void and is only run once during the lifetime of the application. * * @param boolean $forceGenerate - * @param string $phase + * @param string $phase * @param boolean $verboseEnabled - * @param string $debugLevel + * @param string $debugLevel + * @param boolean $allowSkipped * @return void + * @throws TestFrameworkException */ - public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel) + public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = - new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel); + new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped); } } @@ -113,6 +127,7 @@ public static function create($forceGenerate, $phase, $verboseEnabled, $debugLev * This function returns an instance of the MftfApplicationConfig which is created once the application starts. * * @return MftfApplicationConfig + * @throws TestFrameworkException */ public static function getConfig() { @@ -157,6 +172,16 @@ public function getDebugLevel() return $this->debugLevel ?? getenv('MFTF_DEBUG'); } + /** + * Returns a boolean indicating whether mftf is generating skipped tests. + * + * @return boolean + */ + public function allowSkipped() + { + return $this->allowSkipped ?? getenv('ALLOW_SKIPPED'); + } + /** * Returns a string which indicates the phase of mftf execution. * diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index d4044c74b..db175ff3f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -28,6 +28,16 @@ protected function configure() 'r', InputOption::VALUE_NONE, 'remove previous generated suites and tests' + )->addOption( + "force", + 'f', + InputOption::VALUE_NONE, + 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + "allowSkipped", + 'a', + InputOption::VALUE_NONE, + 'Allows MFTF to generate and run skipped tests.' ); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 23651b492..4b4af5855 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -36,11 +36,6 @@ protected function configure() 'name(s) of specific tests to generate' )->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default') ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'Force generation of tests regardless of Magento Instance Configuration' - )->addOption( 'time', 'i', InputOption::VALUE_REQUIRED, @@ -83,6 +78,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); + $allowSkipped = $input->getOption('allowSkipped'); if ($json !== null && !json_decode($json)) { // stop execution if we have failed to properly parse any json passed in by the user @@ -100,7 +96,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ($debug !== MftfApplicationConfig::LEVEL_NONE)); } - $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose); + $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped); // create our manifest file here $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); @@ -128,18 +124,20 @@ protected function execute(InputInterface $input, OutputInterface $output) * @param boolean $force * @param string $debug * @param boolean $verbose + * @param boolean $allowSkipped * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose) + private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose, bool $allowSkipped) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( $force, MftfApplicationConfig::GENERATION_PHASE, $verbose, - $debug + $debug, + $allowSkipped ); $testConfiguration = []; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 76a8bc8f4..4bd64dec3 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -33,11 +33,6 @@ protected function configure() "name of tests to generate and execute" )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test") ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' - )->addOption( 'debug', 'd', InputOption::VALUE_OPTIONAL, @@ -66,6 +61,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); + if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -83,7 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]), '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index d6a580d52..bff232c0c 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -76,13 +76,17 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); + // Create Mftf Configuration MftfApplicationConfig::create( - false, + $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); $testConfiguration = $this->getFailedTestList(); @@ -96,9 +100,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $args = [ '--tests' => $testConfiguration, '--remove' => true, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; - $command->run(new ArrayInput($args), $output); $testManifestList = $this->readTestManifestFile(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index d714a1dd1..dfbeda211 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -34,11 +34,6 @@ protected function configure() 'k', InputOption::VALUE_NONE, "only execute a group of tests without generating from source xml" - )->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' )->addOption( 'debug', 'd', @@ -72,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -85,7 +81,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); if (!$skipGeneration) { @@ -95,7 +92,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--tests' => $testConfiguration, '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 3f1417fd1..d541a5eaf 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Util; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; @@ -1604,7 +1605,7 @@ private function generateTestPhp($test) $testName = str_replace(' ', '', $testName); $testAnnotations = $this->generateAnnotationsPhp($test->getAnnotations(), true); $dependencies = 'AcceptanceTester $I'; - if ($test->isSkipped()) { + if ($test->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $skipString = "This test is skipped due to the following issues:\\n"; $issues = $test->getAnnotations()['skip'] ?? null; if (isset($issues)) { From 9dfcefb884278d07c9e08a94059b3658883a52d4 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 08:59:08 -0500 Subject: [PATCH 2/7] MQE-1582: Enable Testers To Run Skipped Tests - Unit/Static check fixes --- dev/tests/_bootstrap.php | 3 ++- .../Config/MftfApplicationConfig.php | 6 +++--- .../Console/BaseGenerateCommand.php | 7 +++++++ .../Console/GenerateTestsCommand.php | 17 ++++++++--------- .../Console/RunTestCommand.php | 11 +---------- .../Console/RunTestGroupCommand.php | 7 ------- .../DataGenerator/Objects/EntityDataObject.php | 2 +- 7 files changed, 22 insertions(+), 31 deletions(-) diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index d9a4208b2..e06e373d4 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -32,7 +32,8 @@ true, \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE, true, - \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE + \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE, + false ); // Load needed framework env params diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 799798640..6faf322bc 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -55,7 +55,7 @@ class MftfApplicationConfig /** * Boolean which allows MFTF to fully generate skipped tests - * @var bool + * @var boolean */ private $allowSkipped; @@ -108,9 +108,9 @@ private function __construct( * returns void and is only run once during the lifetime of the application. * * @param boolean $forceGenerate - * @param string $phase + * @param string $phase * @param boolean $verboseEnabled - * @param string $debugLevel + * @param string $debugLevel * @param boolean $allowSkipped * @return void * @throws TestFrameworkException diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index db175ff3f..9f989f956 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -38,6 +38,13 @@ protected function configure() 'a', InputOption::VALUE_NONE, 'Allows MFTF to generate and run skipped tests.' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 4b4af5855..98ec87e11 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -46,13 +46,6 @@ protected function configure() 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration' - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -129,8 +122,14 @@ protected function execute(InputInterface $input, OutputInterface $output) * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose, bool $allowSkipped) - { + private function createTestConfiguration( + $json, + array $tests, + bool $force, + $debug, + bool $verbose, + bool $allowSkipped + ) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( $force, diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 4bd64dec3..fab65a8e6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -31,15 +31,7 @@ protected function configure() 'name', InputArgument::REQUIRED | InputArgument::IS_ARRAY, "name of tests to generate and execute" - )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test") - ->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test"); parent::configure(); } @@ -63,7 +55,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allowSkipped'); - if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time throw new TestFrameworkException( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index dfbeda211..81a6b864f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -34,13 +34,6 @@ protected function configure() 'k', InputOption::VALUE_NONE, "only execute a group of tests without generating from source xml" - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT )->addArgument( 'groups', InputArgument::IS_ARRAY | InputArgument::REQUIRED, diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 110869187..34edeee1d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -206,7 +206,7 @@ public function getDataByName($name, $uniquenessFormat) } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); } elseif (array_key_exists($name, $this->data)) { - if (is_array($this->data[$name_lower])) { + if (is_array($this->data[$name])) { return $this->data[$name]; } $this->data[$name] = $dataReferenceResolver->getDataReference( From a9d19170fccaa0ea75e72c44b5de8be5e9a0750e Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 09:42:34 -0500 Subject: [PATCH 3/7] MQE-1582: Enable Testers To Run Skipped Tests - Added new unit test for TestGenerator, also changed visibility of function for testing --- .../Util/TestGeneratorTest.php | 50 +++++++++++++++++++ .../Console/BaseGenerateCommand.php | 1 + .../Util/TestGenerator.php | 4 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index 097150888..c11a40d2c 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -12,6 +12,9 @@ use Magento\FunctionalTestingFramework\Test\Objects\TestObject; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\ObjectManager; +use Magento\FunctionalTestingFramework\ObjectManagerFactory; class TestGeneratorTest extends MagentoTestCase { @@ -38,4 +41,51 @@ public function testEntityException() $testGeneratorObject->createAllTestFiles(null, []); } + + /** + * Tests that skipped tests do not have a fully generated body + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testSkippedNoGeneration() + { + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertContains('This test is skipped', $output); + $this->assertNotContains( $actionInput, $output); + } + + /** + * Tests that skipped tests have a fully generated body when --allowSkipped is passed in + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testAllowSkipped() + { + // Mock allowSkipped for TestGenerator + AspectMock::double(MftfApplicationConfig::class, ['allowSkipped' => true]); + + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertNotContains('This test is skipped', $output); + $this->assertContains($actionInput, $output); + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 9f989f956..6acbf8b4d 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; class BaseGenerateCommand extends Command { diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index d541a5eaf..94868031f 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -228,14 +228,14 @@ public function createAllTestFiles($testManifest = null, $testsToIgnore = null) * @throws TestReferenceException * @throws \Exception */ - private function assembleTestPhp($testObject) + public function assembleTestPhp($testObject) { $usePhp = $this->generateUseStatementsPhp(); $classAnnotationsPhp = $this->generateAnnotationsPhp($testObject->getAnnotations()); $className = $testObject->getCodeceptionName(); try { - if (!$testObject->isSkipped()) { + if (!$testObject->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $hookPhp = $this->generateHooksPhp($testObject->getHooks()); } else { $hookPhp = null; From 3409ac8fbb690fd00c6333ee978988cf3ade1c21 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 09:43:16 -0500 Subject: [PATCH 4/7] MQE-1582: Enable Testers To Run Skipped Tests - Unit Test Cleanup --- .../Magento/FunctionalTestFramework/Util/TestGeneratorTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index c11a40d2c..b5c8537aa 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -13,8 +13,6 @@ use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; -use Magento\FunctionalTestingFramework\ObjectManager; -use Magento\FunctionalTestingFramework\ObjectManagerFactory; class TestGeneratorTest extends MagentoTestCase { From 75cc720af7171c9afe245efd86f8622daad0af58 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 22 Aug 2019 11:43:56 -0500 Subject: [PATCH 5/7] MQE-1582: Enable Testers To Run Skipped Tests - static check fix --- .../Magento/FunctionalTestFramework/Util/TestGeneratorTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index b5c8537aa..cadc1c6a8 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -59,7 +59,7 @@ public function testSkippedNoGeneration() $output = $testGeneratorObject->assembleTestPhp($testObject); $this->assertContains('This test is skipped', $output); - $this->assertNotContains( $actionInput, $output); + $this->assertNotContains($actionInput, $output); } /** From 60fc5715ab9d8042de57283c75fb08fa0acf1760 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 23 Aug 2019 11:30:19 -0500 Subject: [PATCH 6/7] MQE-1582: Enable Testers To Run Skipped Tests - CR Fixes --- .../Console/BaseGenerateCommand.php | 4 ++-- .../Console/GenerateTestsCommand.php | 4 ++-- .../Console/RunTestFailedCommand.php | 10 +--------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 6acbf8b4d..75c521379 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -33,7 +33,7 @@ protected function configure() "force", 'f', InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' + 'force generation and running of tests regardless of Magento Instance Configuration' )->addOption( "allowSkipped", 'a', @@ -43,7 +43,7 @@ protected function configure() 'debug', 'd', InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + 'Run extra validation when generating and running tests. Use option \'none\' to turn off debugging -- added for backward compatibility, will be removed in the next MAJOR release', MftfApplicationConfig::LEVEL_DEFAULT ); diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 98ec87e11..581688615 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -123,10 +123,10 @@ protected function execute(InputInterface $input, OutputInterface $output) * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ private function createTestConfiguration( - $json, + string $json, array $tests, bool $force, - $debug, + string $debug, bool $verbose, bool $allowSkipped ) { diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index bff232c0c..ed00322ff 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -50,15 +50,7 @@ class RunTestFailedCommand extends BaseGenerateCommand protected function configure() { $this->setName('run:failed') - ->setDescription('Execute a set of tests referenced via failed file') - ->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + ->setDescription('Execute a set of tests referenced via failed file'); parent::configure(); } From 51fc64f40f825e485410bd2cf7ff7d5d17349975 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 23 Aug 2019 14:10:16 -0500 Subject: [PATCH 7/7] MQE-1582: Enable Testers To Run Skipped Tests - Added documentation for new argument --- docs/commands/mftf.md | 1 + etc/config/.env.example | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 2301d6f44..aea53c96f 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -122,6 +122,7 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove] | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| +| `--allow-skipped` | Allows MFTF to generate and run tests marked with `.`| | `--debug or --debug=[]`| Performs schema validations on XML files.
DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered.
DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.

NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.
| | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.| diff --git a/etc/config/.env.example b/etc/config/.env.example index 369fd946b..f4aa90e51 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -49,6 +49,9 @@ MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_C #*** Bool property which allows the user to toggle debug output during test execution #MFTF_DEBUG= +#*** Bool property which allows the user to generate and run tests marked as skipped +#ALLOW_SKIPPED=true + #*** Default timeout for wait actions #WAIT_TIMEOUT=10 #*** End of .env ***#