MQE-1582: Enable Testers To Run Skipped Tests

dev/tests/_bootstrap.php

@@ -32,7 +32,8 @@
     true,
     \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
     true,
-    \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE
+    \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE,
+    false
 );
 
 // Load needed framework env params

dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

@@ -12,6 +12,7 @@
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
 use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 
 class TestGeneratorTest extends MagentoTestCase
 {
@@ -38,4 +39,51 @@ public function testEntityException()
 
         $testGeneratorObject->createAllTestFiles(null, []);
     }
+
+    /**
+     * Tests that skipped tests do not have a fully generated body
+     *
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testSkippedNoGeneration()
+    {
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => $actionInput
+        ]);
+
+        $annotations = ['skip' => ['issue']];
+        $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename");
+
+        $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]);
+        $output = $testGeneratorObject->assembleTestPhp($testObject);
+
+        $this->assertContains('This test is skipped', $output);
+        $this->assertNotContains($actionInput, $output);
+    }
+
+    /**
+     * Tests that skipped tests have a fully generated body when --allowSkipped is passed in
+     *
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testAllowSkipped()
+    {
+        // Mock allowSkipped for TestGenerator
+        AspectMock::double(MftfApplicationConfig::class, ['allowSkipped' => true]);
+
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => $actionInput
+        ]);
+
+        $annotations = ['skip' => ['issue']];
+        $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename");
+
+        $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]);
+        $output = $testGeneratorObject->assembleTestPhp($testObject);
+
+        $this->assertNotContains('This test is skipped', $output);
+        $this->assertContains($actionInput, $output);
+    }
 }

docs/commands/mftf.md

@@ -122,6 +122,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|
+| `--allow-skipped` | Allows MFTF to generate and run tests marked with `<skip>.`|
 | `--debug or --debug=[<none>]`| Performs schema validations on XML files. <br/> DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. <br/> DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.</br><br/> NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.</br>|
 | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.|
 

etc/config/.env.example

@@ -49,6 +49,9 @@ MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_C
 #*** Bool property which allows the user to toggle debug output during test execution
 #MFTF_DEBUG=
 
+#*** Bool property which allows the user to generate and run tests marked as skipped
+#ALLOW_SKIPPED=true
+
 #*** Default timeout for wait actions
 #WAIT_TIMEOUT=10
 #*** End of .env ***#

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -9,6 +9,9 @@
 
 class MftfApplicationConfig
 {
+    /**
+     * MFTF Execution Phases
+     */
     const GENERATION_PHASE = "generation";
     const EXECUTION_PHASE = "execution";
     const UNIT_TEST_PHASE = "testing";
@@ -50,6 +53,12 @@ class MftfApplicationConfig
      */
     private $debugLevel;
 
+    /**
+     * Boolean which allows MFTF to fully generate skipped tests
+     * @var boolean
+     */
+    private $allowSkipped;
+
     /**
      * MftfApplicationConfig Singelton Instance
      *
@@ -64,13 +73,15 @@ class MftfApplicationConfig
      * @param string  $phase
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
+     * @param boolean $allowSkipped
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
-        $debugLevel = self::LEVEL_NONE
+        $debugLevel = self::LEVEL_NONE,
+        $allowSkipped = false
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -89,6 +100,7 @@ private function __construct(
             default:
                 $this->debugLevel = self::LEVEL_DEVELOPER;
         }
+        $this->allowSkipped = $allowSkipped;
     }
 
     /**
@@ -99,20 +111,23 @@ private function __construct(
      * @param string  $phase
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
+     * @param boolean $allowSkipped
      * @return void
+     * @throws TestFrameworkException
      */
-    public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
+    public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped)
     {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
+                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
         }
     }
 
     /**
      * This function returns an instance of the MftfApplicationConfig which is created once the application starts.
      *
      * @return MftfApplicationConfig
+     * @throws TestFrameworkException
      */
     public static function getConfig()
     {
@@ -157,6 +172,16 @@ public function getDebugLevel()
         return $this->debugLevel ?? getenv('MFTF_DEBUG');
     }
 
+    /**
+     * Returns a boolean indicating whether mftf is generating skipped tests.
+     *
+     * @return boolean
+     */
+    public function allowSkipped()
+    {
+        return $this->allowSkipped ?? getenv('ALLOW_SKIPPED');
+    }
+
     /**
      * Returns a string which indicates the phase of mftf execution.
      *

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -13,6 +13,7 @@
 use Symfony\Component\Console\Output\OutputInterface;
 use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 
 class BaseGenerateCommand extends Command
 {
@@ -28,6 +29,23 @@ protected function configure()
             'r',
             InputOption::VALUE_NONE,
             'remove previous generated suites and tests'
+        )->addOption(
+            "force",
+            'f',
+            InputOption::VALUE_NONE,
+            'force generation and running of tests regardless of Magento Instance Configuration'
+        )->addOption(
+            "allowSkipped",
+            'a',
+            InputOption::VALUE_NONE,
+            'Allows MFTF to generate and run skipped tests.'
+        )->addOption(
+            'debug',
+            'd',
+            InputOption::VALUE_OPTIONAL,
+            'Run extra validation when generating and running tests. Use option \'none\' to turn off debugging -- 
+             added for backward compatibility, will be removed in the next MAJOR release',
+            MftfApplicationConfig::LEVEL_DEFAULT
         );
     }
 

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -36,11 +36,6 @@ protected function configure()
                 'name(s) of specific tests to generate'
             )->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
             ->addOption(
-                "force",
-                'f',
-                InputOption::VALUE_NONE,
-                'Force generation of tests regardless of Magento Instance Configuration'
-            )->addOption(
                 'time',
                 'i',
                 InputOption::VALUE_REQUIRED,
@@ -51,13 +46,6 @@ protected function configure()
                 't',
                 InputOption::VALUE_REQUIRED,
                 'A parameter accepting a JSON string used to determine the test configuration'
-            )->addOption(
-                'debug',
-                'd',
-                InputOption::VALUE_OPTIONAL,
-                'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- 
-                 added for backward compatibility, will be removed in the next MAJOR release',
-                MftfApplicationConfig::LEVEL_DEFAULT
             );
 
         parent::configure();
@@ -83,6 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
         $remove = $input->getOption('remove');
         $verbose = $output->isVerbose();
+        $allowSkipped = $input->getOption('allowSkipped');
 
         if ($json !== null && !json_decode($json)) {
             // stop execution if we have failed to properly parse any json passed in by the user
@@ -100,7 +89,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 ($debug !== MftfApplicationConfig::LEVEL_NONE));
         }
 
-        $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
+        $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped);
 
         // create our manifest file here
         $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
@@ -128,18 +117,26 @@ protected function execute(InputInterface $input, OutputInterface $output)
      * @param boolean $force
      * @param string  $debug
      * @param boolean $verbose
+     * @param boolean $allowSkipped
      * @return array
      * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
-    private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
-    {
+    private function createTestConfiguration(
+        string $json,
+        array $tests,
+        bool $force,
+        string $debug,
+        bool $verbose,
+        bool $allowSkipped
+    ) {
         // set our application configuration so we can references the user options in our framework
         MftfApplicationConfig::create(
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             $verbose,
-            $debug
+            $debug,
+            $allowSkipped
         );
 
         $testConfiguration = [];

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -31,20 +31,7 @@ protected function configure()
                 'name',
                 InputArgument::REQUIRED | InputArgument::IS_ARRAY,
                 "name of tests to generate and execute"
-            )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test")
-            ->addOption(
-                "force",
-                'f',
-                InputOption::VALUE_NONE,
-                'force generation of tests regardless of Magento Instance Configuration'
-            )->addOption(
-                'debug',
-                'd',
-                InputOption::VALUE_OPTIONAL,
-                'Run extra validation when running tests. Use option \'none\' to turn off debugging -- 
-                 added for backward compatibility, will be removed in the next MAJOR release',
-                MftfApplicationConfig::LEVEL_DEFAULT
-            );
+            )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test");
 
         parent::configure();
     }
@@ -66,6 +53,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $force = $input->getOption('force');
         $remove = $input->getOption('remove');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
+        $allowSkipped = $input->getOption('allowSkipped');
 
         if ($skipGeneration and $remove) {
             // "skip-generate" and "remove" options cannot be used at the same time
@@ -83,7 +71,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 ]),
                 '--force' => $force,
                 '--remove' => $remove,
-                '--debug' => $debug
+                '--debug' => $debug,
+                '--allowSkipped' => $allowSkipped
             ];
             $command->run(new ArrayInput($args), $output);
         }

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -50,15 +50,7 @@ class RunTestFailedCommand extends BaseGenerateCommand
     protected function configure()
     {
         $this->setName('run:failed')
-            ->setDescription('Execute a set of tests referenced via failed file')
-            ->addOption(
-                'debug',
-                'd',
-                InputOption::VALUE_OPTIONAL,
-                'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- 
-                 added for backward compatibility, will be removed in the next MAJOR release',
-                MftfApplicationConfig::LEVEL_DEFAULT
-            );
+            ->setDescription('Execute a set of tests referenced via failed file');
 
         parent::configure();
     }
@@ -76,13 +68,17 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        $force = $input->getOption('force');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
+        $allowSkipped = $input->getOption('allowSkipped');
+
         // Create Mftf Configuration
         MftfApplicationConfig::create(
-            false,
+            $force,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
-            $debug
+            $debug,
+            $allowSkipped
         );
 
         $testConfiguration = $this->getFailedTestList();
@@ -96,9 +92,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $args = [
             '--tests' => $testConfiguration,
             '--remove' => true,
-            '--debug' => $debug
+            '--debug' => $debug,
+            '--allowSkipped' => $allowSkipped
         ];
-
         $command->run(new ArrayInput($args), $output);
 
         $testManifestList = $this->readTestManifestFile();