diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php index 1b1686ce2..234f677e8 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php @@ -156,6 +156,24 @@ public function testThreeTestOneSuiteOneGroupMix() $this->assertEquals($expected, $actual); } + public function testSuiteToTestSyntax() + { + $testOne = new TestObject('Test1', [], [], []); + $suiteOne = new SuiteObject( + 'Suite1', + ['Test1' => $testOne], + [], + [] + ); + + $testArray = ['Test1' => $testOne]; + $suiteArray = ['Suite1' => $suiteOne]; + $this->mockHandlers($testArray, $suiteArray); + $actual = json_decode($this->callTestConfig(['Suite1:Test1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]]; + $this->assertEquals($expected, $actual); + } + /** * Mock handlers to skip parsing * @param array $testArray diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php index 29158b0f5..e3f820ede 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php @@ -6,6 +6,7 @@ namespace Tests\unit\Magento\FunctionalTestFramework\Suite; use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Suite\SuiteGenerator; @@ -17,6 +18,7 @@ use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser; use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; +use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory; use tests\unit\Util\SuiteDataArrayBuilder; use tests\unit\Util\TestDataArrayBuilder; use tests\unit\Util\TestLoggingUtil; @@ -143,6 +145,73 @@ public function testGenerateEmptySuite() $mockSuiteGenerator->generateSuite("basicTestSuite"); } + public function testInvalidSuiteTestPair() + { + // Mock Suite1 => Test1 and Suite2 => Test2 + $suiteDataArrayBuilder = new SuiteDataArrayBuilder(); + $mockData = $suiteDataArrayBuilder + ->withName('Suite1') + ->includeGroups(['group1']) + ->build(); + $suiteDataArrayBuilder = new SuiteDataArrayBuilder(); + $mockData2 = $suiteDataArrayBuilder + ->withName('Suite2') + ->includeGroups(['group2']) + ->build(); + $mockSuiteData = array_merge_recursive($mockData, $mockData2); + + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest = $testDataArrayBuilder + ->withName('Test1') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest2 = $testDataArrayBuilder + ->withName('Test2') + ->withAnnotations(['group' => [['value' => 'group2']]]) + ->withTestActions() + ->build(); + $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)]; + $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData); + + // Make invalid manifest + $suiteConfig = ['Suite2' => ['Test1']]; + $manifest = TestManifestFactory::makeManifest('default', $suiteConfig); + + // Set up Expected Exception + $this->expectException(TestReferenceException::class); + $this->expectExceptionMessageRegExp('(Suite: "Suite2" Tests: "Test1")'); + + // parse and generate suite object with mocked data and manifest + $mockSuiteGenerator = SuiteGenerator::getInstance(); + $mockSuiteGenerator->generateAllSuites($manifest); + } + + public function testNonExistentSuiteTestPair() + { + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest = $testDataArrayBuilder + ->withName('Test1') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + $mockTestData = ['tests' => array_merge($mockSimpleTest)]; + $this->setMockTestAndSuiteParserOutput($mockTestData, []); + + // Make invalid manifest + $suiteConfig = ['Suite3' => ['Test1']]; + $manifest = TestManifestFactory::makeManifest('default', $suiteConfig); + + // Set up Expected Exception + $this->expectException(TestReferenceException::class); + $this->expectExceptionMessageRegExp('#Suite3 is not defined#'); + + // parse and generate suite object with mocked data and manifest + $mockSuiteGenerator = SuiteGenerator::getInstance(); + $mockSuiteGenerator->generateAllSuites($manifest); + } + /** * Function used to set mock for parser return and force init method to run between tests. * diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 2440428f9..642bbb1bd 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -42,6 +42,12 @@ vendor/bin/mftf generate:tests vendor/bin/mftf generate:tests AdminLoginTest StorefrontPersistedCustomerLoginTest ``` +### Generate test by test and suite name + +```bash +vendor/bin/mftf generate:tests LoginSuite:AdminLoginTest +``` + ### Generate and run the tests for a specified group ```bash @@ -58,6 +64,14 @@ vendor/bin/mftf run:test AdminLoginTest StorefrontPersistedCustomerLoginTest -r This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests. +### Generate and run particular test in a specific suite's context + +```bash +vendor/bin/mftf run:test LoginSuite:AdminLoginTest -r +``` + +This command cleans up previously generated tests; generates and run `AdminLoginTest` within the context of the `LoginSuite`. + ### Generate and run a testManifest.txt file ```bash diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 0953e9d68..a0d94b16e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -87,6 +87,10 @@ protected function getTestAndSuiteConfiguration(array $tests) $suiteToTestPair = []; foreach($tests as $test) { + if (strpos($test, ':') !== false) { + $suiteToTestPair[] = $test; + continue; + } if (array_key_exists($test, $testsReferencedInSuites)) { $suites = $testsReferencedInSuites[$test]; foreach ($suites as $suite) { diff --git a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php index 30a67c31d..bbaaf4379 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\Suite\Handlers; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; use Magento\FunctionalTestingFramework\ObjectManagerFactory; @@ -73,7 +74,7 @@ public static function getInstance(): ObjectHandlerInterface public function getObject($objectName): SuiteObject { if (!array_key_exists($objectName, $this->suiteObjects)) { - trigger_error("Suite ${objectName} is not defined.", E_USER_ERROR); + throw new TestReferenceException("Suite ${objectName} is not defined in xml."); } return $this->suiteObjects[$objectName]; } diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index 4df7daa75..e3da10497 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -178,12 +178,14 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o { $suiteRef = $originalSuiteName ?? $suiteName; $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests(); - $errorMsg = "Cannot reference tests whcih are not declared as part of suite."; + $errorMsg = "Cannot reference tests which are not declared as part of suite"; $invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef)); if (!empty($invalidTestRef)) { - throw new TestReferenceException($errorMsg, ['suite' => $suiteRef, 'test' => $invalidTestRef]); + $testList = implode("\", \"", $invalidTestRef); + $fullError = $errorMsg . " (Suite: \"{$suiteRef}\" Tests: \"{$testList}\")"; + throw new TestReferenceException($fullError, ['suite' => $suiteRef, 'test' => $invalidTestRef]); } }