MQE-2008: Filter test generation and execution by severity

dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml

@@ -15,7 +15,7 @@
             <stories value="[Deprecated] MFTF DevDocs available"/>
             <title value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
             <description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
-            <severity value="CRITICAL"/>
+            <severity value="MINOR"/>
             <group value="mftf"/>
         </annotations>
 

dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php

@@ -65,7 +65,7 @@ public function testGetSuiteObject()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+        $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -69,7 +69,7 @@ public function testGenerateSuite()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group2']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
 
         // Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group1']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, []);
 
         // Make invalid manifest

dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

@@ -41,7 +41,7 @@ public function testGetTestObject()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
 
         // run object handler method
         $toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
+        $this->setMockParserOutput(array_merge($includeTest, $excludeTest));
 
         // execute test method
         $toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
 
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
         // Execute Test Method
         $toh = TestObjectHandler::getInstance();
         $actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
             ->build();
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $testOne]);
+        $this->setMockParserOutput($testOne);
 
         $toh = TestObjectHandler::getInstance();
 
@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
+        $this->setMockParserOutput(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
 

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php

@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockExtendedTest)];
+        $mockTestData = array_merge($mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         $this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");

docs/commands/mftf.md

@@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | Option | Description|
 | ---| --- |
 | `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.<br/> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.<br/> Example: `generate:tests --config=parallel`. |
+| `--filter` | Option to filter tests to be generated.<br/>Template: '<filterName>:<filterValue>'.<br/>Existing filter types: severity.<br/>Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.<br/>Example: --filter=severity:CRITICAL|
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -6,6 +6,7 @@
 namespace Magento\FunctionalTestingFramework\Config;
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 
 class MftfApplicationConfig
 {
@@ -25,6 +26,13 @@ class MftfApplicationConfig
     const LEVEL_NONE = "none";
     const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
 
+    /**
+     * Contains object with test filters.
+     *
+     * @var FilterList
+     */
+    private $filterList;
+
     /**
      * Determines whether the user has specified a force option for generation
      *
@@ -74,14 +82,16 @@ class MftfApplicationConfig
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -101,6 +111,7 @@ private function __construct(
                 $this->debugLevel = self::LEVEL_DEVELOPER;
         }
         $this->allowSkipped = $allowSkipped;
+        $this->filterList = new FilterList($filters);
     }
 
     /**
@@ -112,6 +123,7 @@ private function __construct(
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @return void
      * @throws TestFrameworkException
      */
@@ -120,11 +132,19 @@ public static function create(
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
+                new MftfApplicationConfig(
+                    $forceGenerate,
+                    $phase,
+                    $verboseEnabled,
+                    $debugLevel,
+                    $allowSkipped,
+                    $filters
+                );
         }
     }
 
@@ -196,4 +216,14 @@ public function getPhase()
     {
         return $this->phase;
     }
+
+    /**
+     * Returns a class with registered filter list.
+     *
+     * @return FilterList
+     */
+    public function getFilterList()
+    {
+        return $this->filterList;
+    }
 }

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -18,7 +18,11 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
 
+/**
+ * @SuppressWarnings(PHPMD)
+ */
 class GenerateTestsCommand extends BaseGenerateCommand
 {
     /**
@@ -30,6 +34,7 @@ protected function configure()
     {
         $this->setName('generate:tests')
             ->setDescription('Run validation and generate all test files and suites based on xml declarations')
+            ->addUsage('AdminLoginTest')
             ->addArgument(
                 'name',
                 InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -46,6 +51,15 @@ protected function configure()
                 't',
                 InputOption::VALUE_REQUIRED,
                 'A parameter accepting a JSON string used to determine the test configuration'
+            )->addOption(
+                'filter',
+                null,
+                InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
+                'Option to filter tests to be generated.' . PHP_EOL
+                . '<info>Template:</info> <filterName>:<filterValue>' . PHP_EOL
+                . '<info>Existing filter types:</info> severity.' . PHP_EOL
+                . '<info>Existing severity values:</info> BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.' . PHP_EOL
+                . '<info>Example:</info> --filter=severity:CRITICAL' . PHP_EOL
             );
 
         parent::configure();
@@ -56,13 +70,14 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return void
+     * @return void|integer
      * @throws TestFrameworkException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $ioStyle = new SymfonyStyle($input, $output);
         $tests = $input->getArgument('name');
         $config = $input->getOption('config');
         $json = $input->getOption('tests'); // for backward compatibility
@@ -72,15 +87,25 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $remove = $input->getOption('remove');
         $verbose = $output->isVerbose();
         $allowSkipped = $input->getOption('allow-skipped');
-
+        $filters = $input->getOption('filter');
+        foreach ($filters as $filter) {
+            list($filterType, $filterValue) = explode(':', $filter);
+            $filterList[$filterType][] = $filterValue;
+        }
         // Set application configuration so we can references the user options in our framework
-        MftfApplicationConfig::create(
-            $force,
-            MftfApplicationConfig::GENERATION_PHASE,
-            $verbose,
-            $debug,
-            $allowSkipped
-        );
+        try {
+            MftfApplicationConfig::create(
+                $force,
+                MftfApplicationConfig::GENERATION_PHASE,
+                $verbose,
+                $debug,
+                $allowSkipped,
+                $filterList ?? []
+            );
+        } catch (\Exception $exception) {
+            $ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
+            return 1;
+        }
 
         $this->setOutputStyle($input, $output);
         $this->showMftfNotices($output);
@@ -105,20 +130,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 ($debug !== MftfApplicationConfig::LEVEL_NONE));
         }
 
-        $testConfiguration = $this->createTestConfiguration($json, $tests);
+        try {
+            $testConfiguration = $this->createTestConfiguration($json, $tests);
 
-        // create our manifest file here
-        $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
-        TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
+            // create our manifest file here
+            $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
 
-        if ($config == 'parallel') {
-            /** @var ParallelTestManifest $testManifest */
-            $testManifest->createTestGroups($time);
-        }
+            TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
+
+            if ($config == 'parallel') {
+                /** @var ParallelTestManifest $testManifest */
+                $testManifest->createTestGroups($time);
+            }
+
+            SuiteGenerator::getInstance()->generateAllSuites($testManifest);
 
-        SuiteGenerator::getInstance()->generateAllSuites($testManifest);
+            $testManifest->generate();
+        } catch (\Exception $e) {
+            $message = $e->getMessage() . PHP_EOL;
+            $message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
+            $message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
+            $message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
+            $ioStyle->note($message);
 
-        $testManifest->generate();
+            return 1;
+        }
 
         $output->writeln("Generate Tests Command Run");
     }

src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php

@@ -0,0 +1,27 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+declare(strict_types=1);
+
+namespace Magento\FunctionalTestingFramework\Filter;
+
+/**
+ * Interface for future test filters
+ * @api
+ */
+interface FilterInterface
+{
+    /**
+     * @param array $filterValues
+     */
+    public function __construct(array $filterValues = []);
+
+    /**
+     * @param array $tests
+     * @return void
+     */
+    public function filter(array &$tests);
+}