MQE-1713: Generate/run test in single suite context

- New Unit test for SuiteGenerator
- Better message for exception in SuiteGenerator

Author: KevinBKozan

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -6,6 +6,7 @@
 namespace Tests\unit\Magento\FunctionalTestFramework\Suite;
 
 use AspectMock\Test as AspectMock;
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager;
 use Magento\FunctionalTestingFramework\ObjectManagerFactory;
 use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
@@ -17,6 +18,7 @@
 use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
 use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
 use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
 use tests\unit\Util\SuiteDataArrayBuilder;
 use tests\unit\Util\TestDataArrayBuilder;
 use tests\unit\Util\TestLoggingUtil;
@@ -143,6 +145,49 @@ public function testGenerateEmptySuite()
         $mockSuiteGenerator->generateSuite("basicTestSuite");
     }
 
+    public function testInvalidTestRef()
+    {
+        // Mock Suite1 => Test1 and Suite2 => Test2
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockData = $suiteDataArrayBuilder
+            ->withName('Suite1')
+            ->includeGroups(['group1'])
+            ->build();
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockData2 = $suiteDataArrayBuilder
+            ->withName('Suite2')
+            ->includeGroups(['group2'])
+            ->build();
+        $mockSuiteData = array_merge_recursive($mockData, $mockData2);
+
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest = $testDataArrayBuilder
+            ->withName('Test1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest2 = $testDataArrayBuilder
+            ->withName('Test2')
+            ->withAnnotations(['group' => [['value' => 'group2']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
+
+        // Make invalid manifest
+        $suiteConfig = ['Suite2' => ['Test1']];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // Set up Expected Exception
+        $this->expectException(TestReferenceException::class);
+        $this->expectExceptionMessageRegExp('(Suite: "Suite2" Tests: "Test1")');
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -178,12 +178,14 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o
     {
         $suiteRef = $originalSuiteName ?? $suiteName;
         $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
-        $errorMsg = "Cannot reference tests which are not declared as part of suite.";
+        $errorMsg = "Cannot reference tests which are not declared as part of suite";
 
         $invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef));
 
         if (!empty($invalidTestRef)) {
-            throw new TestReferenceException($errorMsg, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
+            $testList = implode("\", \"", $invalidTestRef);
+            $fullError = $errorMsg . " (Suite: \"{$suiteRef}\" Tests: \"{$testList}\")";
+            throw new TestReferenceException($fullError, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
         }
     }