From 995016a42bee7a0043ae2331be968f1f7a91369b Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 8 Jan 2020 14:56:40 -0600 Subject: [PATCH 1/5] MQE-1713: Generate/run test in single suite context - added support for 'suite:test' syntax in generate and run:test command - minor fixes to exceptions in suite mechanisms --- .../Console/BaseGenerateCommandTest.php | 18 ++++++++++++++++++ .../Console/BaseGenerateCommand.php | 4 ++++ .../Suite/Handlers/SuiteObjectHandler.php | 3 ++- .../Suite/SuiteGenerator.php | 2 +- 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php index 1b1686ce2..234f677e8 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php @@ -156,6 +156,24 @@ public function testThreeTestOneSuiteOneGroupMix() $this->assertEquals($expected, $actual); } + public function testSuiteToTestSyntax() + { + $testOne = new TestObject('Test1', [], [], []); + $suiteOne = new SuiteObject( + 'Suite1', + ['Test1' => $testOne], + [], + [] + ); + + $testArray = ['Test1' => $testOne]; + $suiteArray = ['Suite1' => $suiteOne]; + $this->mockHandlers($testArray, $suiteArray); + $actual = json_decode($this->callTestConfig(['Suite1:Test1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]]; + $this->assertEquals($expected, $actual); + } + /** * Mock handlers to skip parsing * @param array $testArray diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 0953e9d68..72cfc8578 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -87,6 +87,10 @@ protected function getTestAndSuiteConfiguration(array $tests) $suiteToTestPair = []; foreach($tests as $test) { + if (strpos($test, ':') !== null) { + $suiteToTestPair[] = $test; + continue; + } if (array_key_exists($test, $testsReferencedInSuites)) { $suites = $testsReferencedInSuites[$test]; foreach ($suites as $suite) { diff --git a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php index 30a67c31d..bbaaf4379 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\Suite\Handlers; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; use Magento\FunctionalTestingFramework\ObjectManagerFactory; @@ -73,7 +74,7 @@ public static function getInstance(): ObjectHandlerInterface public function getObject($objectName): SuiteObject { if (!array_key_exists($objectName, $this->suiteObjects)) { - trigger_error("Suite ${objectName} is not defined.", E_USER_ERROR); + throw new TestReferenceException("Suite ${objectName} is not defined in xml."); } return $this->suiteObjects[$objectName]; } diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index 4df7daa75..077e6d1cc 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -178,7 +178,7 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o { $suiteRef = $originalSuiteName ?? $suiteName; $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests(); - $errorMsg = "Cannot reference tests whcih are not declared as part of suite."; + $errorMsg = "Cannot reference tests which are not declared as part of suite."; $invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef)); From aea7e8d76bc24c69cb506370e8bd8c78ca06e0ac Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 9 Jan 2020 08:58:24 -0600 Subject: [PATCH 2/5] MQE-1713: Generate/run test in single suite contex - Fixed wrong comparison --- .../FunctionalTestingFramework/Console/BaseGenerateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 72cfc8578..a0d94b16e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -87,7 +87,7 @@ protected function getTestAndSuiteConfiguration(array $tests) $suiteToTestPair = []; foreach($tests as $test) { - if (strpos($test, ':') !== null) { + if (strpos($test, ':') !== false) { $suiteToTestPair[] = $test; continue; } From af7c2fb9a4af3de4a9a401768bb6794f149ca711 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 9 Jan 2020 09:03:40 -0600 Subject: [PATCH 3/5] MQE-1713: Generate/run test in single suite context - Documentation update --- docs/commands/mftf.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 2440428f9..642bbb1bd 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -42,6 +42,12 @@ vendor/bin/mftf generate:tests vendor/bin/mftf generate:tests AdminLoginTest StorefrontPersistedCustomerLoginTest ``` +### Generate test by test and suite name + +```bash +vendor/bin/mftf generate:tests LoginSuite:AdminLoginTest +``` + ### Generate and run the tests for a specified group ```bash @@ -58,6 +64,14 @@ vendor/bin/mftf run:test AdminLoginTest StorefrontPersistedCustomerLoginTest -r This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests. +### Generate and run particular test in a specific suite's context + +```bash +vendor/bin/mftf run:test LoginSuite:AdminLoginTest -r +``` + +This command cleans up previously generated tests; generates and run `AdminLoginTest` within the context of the `LoginSuite`. + ### Generate and run a testManifest.txt file ```bash From 22e1f48404a6fc2a68d152aec749db246d1a3428 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 14 Jan 2020 09:46:50 -0600 Subject: [PATCH 4/5] MQE-1713: Generate/run test in single suite context - New Unit test for SuiteGenerator - Better message for exception in SuiteGenerator --- .../Suite/SuiteGeneratorTest.php | 45 +++++++++++++++++++ .../Suite/SuiteGenerator.php | 6 ++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php index 29158b0f5..eded50b41 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php @@ -6,6 +6,7 @@ namespace Tests\unit\Magento\FunctionalTestFramework\Suite; use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Suite\SuiteGenerator; @@ -17,6 +18,7 @@ use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser; use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; +use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory; use tests\unit\Util\SuiteDataArrayBuilder; use tests\unit\Util\TestDataArrayBuilder; use tests\unit\Util\TestLoggingUtil; @@ -143,6 +145,49 @@ public function testGenerateEmptySuite() $mockSuiteGenerator->generateSuite("basicTestSuite"); } + public function testInvalidTestRef() + { + // Mock Suite1 => Test1 and Suite2 => Test2 + $suiteDataArrayBuilder = new SuiteDataArrayBuilder(); + $mockData = $suiteDataArrayBuilder + ->withName('Suite1') + ->includeGroups(['group1']) + ->build(); + $suiteDataArrayBuilder = new SuiteDataArrayBuilder(); + $mockData2 = $suiteDataArrayBuilder + ->withName('Suite2') + ->includeGroups(['group2']) + ->build(); + $mockSuiteData = array_merge_recursive($mockData, $mockData2); + + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest = $testDataArrayBuilder + ->withName('Test1') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest2 = $testDataArrayBuilder + ->withName('Test2') + ->withAnnotations(['group' => [['value' => 'group2']]]) + ->withTestActions() + ->build(); + $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)]; + $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData); + + // Make invalid manifest + $suiteConfig = ['Suite2' => ['Test1']]; + $manifest = TestManifestFactory::makeManifest('default', $suiteConfig); + + // Set up Expected Exception + $this->expectException(TestReferenceException::class); + $this->expectExceptionMessageRegExp('(Suite: "Suite2" Tests: "Test1")'); + + // parse and generate suite object with mocked data and manifest + $mockSuiteGenerator = SuiteGenerator::getInstance(); + $mockSuiteGenerator->generateAllSuites($manifest); + } + /** * Function used to set mock for parser return and force init method to run between tests. * diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index 077e6d1cc..e3da10497 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -178,12 +178,14 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o { $suiteRef = $originalSuiteName ?? $suiteName; $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests(); - $errorMsg = "Cannot reference tests which are not declared as part of suite."; + $errorMsg = "Cannot reference tests which are not declared as part of suite"; $invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef)); if (!empty($invalidTestRef)) { - throw new TestReferenceException($errorMsg, ['suite' => $suiteRef, 'test' => $invalidTestRef]); + $testList = implode("\", \"", $invalidTestRef); + $fullError = $errorMsg . " (Suite: \"{$suiteRef}\" Tests: \"{$testList}\")"; + throw new TestReferenceException($fullError, ['suite' => $suiteRef, 'test' => $invalidTestRef]); } } From ce8b333b0d4cd115b69f42b9f2a2f5e08b3eef89 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 14 Jan 2020 11:11:26 -0600 Subject: [PATCH 5/5] MQE-1713: Generate/run test in single suite context - Added unit test for different exception message --- .../Suite/SuiteGeneratorTest.php | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php index eded50b41..e3f820ede 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php @@ -145,7 +145,7 @@ public function testGenerateEmptySuite() $mockSuiteGenerator->generateSuite("basicTestSuite"); } - public function testInvalidTestRef() + public function testInvalidSuiteTestPair() { // Mock Suite1 => Test1 and Suite2 => Test2 $suiteDataArrayBuilder = new SuiteDataArrayBuilder(); @@ -188,6 +188,30 @@ public function testInvalidTestRef() $mockSuiteGenerator->generateAllSuites($manifest); } + public function testNonExistentSuiteTestPair() + { + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest = $testDataArrayBuilder + ->withName('Test1') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + $mockTestData = ['tests' => array_merge($mockSimpleTest)]; + $this->setMockTestAndSuiteParserOutput($mockTestData, []); + + // Make invalid manifest + $suiteConfig = ['Suite3' => ['Test1']]; + $manifest = TestManifestFactory::makeManifest('default', $suiteConfig); + + // Set up Expected Exception + $this->expectException(TestReferenceException::class); + $this->expectExceptionMessageRegExp('#Suite3 is not defined#'); + + // parse and generate suite object with mocked data and manifest + $mockSuiteGenerator = SuiteGenerator::getInstance(); + $mockSuiteGenerator->generateAllSuites($manifest); + } + /** * Function used to set mock for parser return and force init method to run between tests. *