Merge branch 'develop' into MQE-1761

Author: soumyau

dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php

@@ -156,6 +156,24 @@ public function testThreeTestOneSuiteOneGroupMix()
         $this->assertEquals($expected, $actual);
     }
 
+    public function testSuiteToTestSyntax()
+    {
+        $testOne = new TestObject('Test1', [], [], []);
+        $suiteOne = new SuiteObject(
+            'Suite1',
+            ['Test1' => $testOne],
+            [],
+            []
+        );
+
+        $testArray = ['Test1' => $testOne];
+        $suiteArray = ['Suite1' => $suiteOne];
+        $this->mockHandlers($testArray, $suiteArray);
+        $actual = json_decode($this->callTestConfig(['Suite1:Test1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
+        $this->assertEquals($expected, $actual);
+    }
+
     /**
      * Mock handlers to skip parsing
      * @param array $testArray

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -6,6 +6,7 @@
 namespace Tests\unit\Magento\FunctionalTestFramework\Suite;
 
 use AspectMock\Test as AspectMock;
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager;
 use Magento\FunctionalTestingFramework\ObjectManagerFactory;
 use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
@@ -17,6 +18,7 @@
 use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
 use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
 use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
 use tests\unit\Util\SuiteDataArrayBuilder;
 use tests\unit\Util\TestDataArrayBuilder;
 use tests\unit\Util\TestLoggingUtil;
@@ -143,6 +145,73 @@ public function testGenerateEmptySuite()
         $mockSuiteGenerator->generateSuite("basicTestSuite");
     }
 
+    public function testInvalidSuiteTestPair()
+    {
+        // Mock Suite1 => Test1 and Suite2 => Test2
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockData = $suiteDataArrayBuilder
+            ->withName('Suite1')
+            ->includeGroups(['group1'])
+            ->build();
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockData2 = $suiteDataArrayBuilder
+            ->withName('Suite2')
+            ->includeGroups(['group2'])
+            ->build();
+        $mockSuiteData = array_merge_recursive($mockData, $mockData2);
+
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest = $testDataArrayBuilder
+            ->withName('Test1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest2 = $testDataArrayBuilder
+            ->withName('Test2')
+            ->withAnnotations(['group' => [['value' => 'group2']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
+
+        // Make invalid manifest
+        $suiteConfig = ['Suite2' => ['Test1']];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // Set up Expected Exception
+        $this->expectException(TestReferenceException::class);
+        $this->expectExceptionMessageRegExp('(Suite: "Suite2" Tests: "Test1")');
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+    }
+
+    public function testNonExistentSuiteTestPair()
+    {
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest = $testDataArrayBuilder
+            ->withName('Test1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $this->setMockTestAndSuiteParserOutput($mockTestData, []);
+
+        // Make invalid manifest
+        $suiteConfig = ['Suite3' => ['Test1']];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // Set up Expected Exception
+        $this->expectException(TestReferenceException::class);
+        $this->expectExceptionMessageRegExp('#Suite3 is not defined#');
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *

docs/commands/mftf.md

@@ -42,6 +42,12 @@ vendor/bin/mftf generate:tests
 vendor/bin/mftf generate:tests AdminLoginTest StorefrontPersistedCustomerLoginTest
 ```
 
+### Generate test by test and suite name
+
+```bash
+vendor/bin/mftf generate:tests LoginSuite:AdminLoginTest
+```
+
 ### Generate and run the tests for a specified group
 
 ```bash
@@ -58,6 +64,14 @@ vendor/bin/mftf run:test AdminLoginTest StorefrontPersistedCustomerLoginTest -r
 
 This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests.
 
+### Generate and run particular test in a specific suite's context
+
+```bash
+vendor/bin/mftf run:test LoginSuite:AdminLoginTest -r
+```
+
+This command cleans up previously generated tests; generates and run `AdminLoginTest` within the context of the `LoginSuite`.
+
 ### Generate and run a testManifest.txt file
 
 ```bash

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -87,6 +87,10 @@ protected function getTestAndSuiteConfiguration(array $tests)
         $suiteToTestPair = [];
 
         foreach($tests as $test) {
+            if (strpos($test, ':') !== false) {
+                $suiteToTestPair[] = $test;
+                continue;
+            }
             if (array_key_exists($test, $testsReferencedInSuites)) {
                 $suites = $testsReferencedInSuites[$test];
                 foreach ($suites as $suite) {

src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

@@ -5,6 +5,7 @@
  */
 namespace Magento\FunctionalTestingFramework\Suite\Handlers;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface;
 use Magento\FunctionalTestingFramework\ObjectManagerFactory;
@@ -73,7 +74,7 @@ public static function getInstance(): ObjectHandlerInterface
     public function getObject($objectName): SuiteObject
     {
         if (!array_key_exists($objectName, $this->suiteObjects)) {
-            trigger_error("Suite ${objectName} is not defined.", E_USER_ERROR);
+            throw new TestReferenceException("Suite ${objectName} is not defined in xml.");
         }
         return $this->suiteObjects[$objectName];
     }

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -178,12 +178,14 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o
     {
         $suiteRef = $originalSuiteName ?? $suiteName;
         $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
-        $errorMsg = "Cannot reference tests whcih are not declared as part of suite.";
+        $errorMsg = "Cannot reference tests which are not declared as part of suite";
 
         $invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef));
 
         if (!empty($invalidTestRef)) {
-            throw new TestReferenceException($errorMsg, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
+            $testList = implode("\", \"", $invalidTestRef);
+            $fullError = $errorMsg . " (Suite: \"{$suiteRef}\" Tests: \"{$testList}\")";
+            throw new TestReferenceException($fullError, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
         }
     }