From c9cbeb60d7237c55228ac571e3afc836b8018df2 Mon Sep 17 00:00:00 2001 From: Ian Meron Date: Wed, 4 Apr 2018 10:40:42 -0500 Subject: [PATCH 1/2] MQE-878: Modify parallel grouping algorithm to work with suites - add new parallel group sorter - update suite object handler to work with new parallel groupings - alter manifest files to use new -g notation for groups - add new unit tests for new classes --- .../Suite/Handlers/SuiteObjectHandlerTest.php | 109 +++++++ .../Util/Sorter/ParallelGroupSorterTest.php | 122 ++++++++ .../Validation/NameValidationUtilTest.php} | 6 +- dev/tests/unit/Util/SuiteDataArrayBuilder.php | 234 +++++++++++++++ .../Tests/SuiteGenerationTest.php | 1 + .../Suite/Generators/GroupClassGenerator.php | 16 +- .../Suite/Handlers/SuiteObjectHandler.php | 4 - .../Suite/Objects/SuiteObject.php | 14 +- .../Suite/SuiteGenerator.php | 140 ++++++--- .../Suite/Util/SuiteObjectExtractor.php | 4 + .../Test/Util/TestNameValidationUtil.php | 61 ---- .../Test/Util/TestObjectExtractor.php | 3 +- .../Util/Manifest/BaseTestManifest.php | 3 +- .../Util/Manifest/DefaultTestManifest.php | 26 +- .../Util/Manifest/ParallelTestManifest.php | 75 +++-- .../Util/Manifest/SingleRunTestManifest.php | 4 +- .../Util/Manifest/TestManifestFactory.php | 4 + .../Util/Sorter/ParallelGroupSorter.php | 270 ++++++++++++++++++ .../Util/TestGenerator.php | 16 +- .../Util/Validation/NameValidationUtil.php | 53 ++++ 20 files changed, 1014 insertions(+), 151 deletions(-) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php rename dev/tests/unit/Magento/FunctionalTestFramework/{Test/Util/TestNameValidationUtilTest.php => Util/Validation/NameValidationUtilTest.php} (90%) create mode 100644 dev/tests/unit/Util/SuiteDataArrayBuilder.php delete mode 100644 src/Magento/FunctionalTestingFramework/Test/Util/TestNameValidationUtil.php create mode 100644 src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php create mode 100644 src/Magento/FunctionalTestingFramework/Util/Validation/NameValidationUtil.php diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php new file mode 100644 index 000000000..ede99bf34 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php @@ -0,0 +1,109 @@ +withName('basicTestSuite') + ->withAfterHook() + ->withBeforeHook() + ->includeTests(['simpleTest']) + ->includeGroups(['group1']) + ->excludeTests(['group1Test2']) + ->excludeGroups(['group2']) + ->build(); + + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockSimpleTest = $testDataArrayBuilder + ->withName('simpleTest') + ->withTestActions() + ->build(); + + $mockGroup1Test1 = $testDataArrayBuilder + ->withName('group1Test1') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + + $mockGroup1Test2 = $testDataArrayBuilder + ->withName('group1Test2') + ->withAnnotations(['group' => [['value' => 'group1']]]) + ->withTestActions() + ->build(); + + $mockGroup2Test1 = $testDataArrayBuilder + ->withName('group2Test1') + ->withAnnotations(['group' => [['value' => 'group2']]]) + ->withTestActions() + ->build(); + + $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)]; + $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData); + + // parse and retrieve suite object with mocked data + $basicTestSuiteObj = SuiteObjectHandler::getInstance()->getObject('basicTestSuite'); + + // assert on created suite object + $this->assertEquals($basicTestSuiteObj->getName(), 'basicTestSuite'); + $this->assertCount(2, $basicTestSuiteObj->getTests()); + $this->assertNotEmpty($basicTestSuiteObj->getBeforeHook()); + $this->assertNotEmpty($basicTestSuiteObj->getAfterHook()); + } + + /** + * Function used to set mock for parser return and force init method to run between tests. + * + * @param array $testData + * @throws \Exception + */ + private function setMockTestAndSuiteParserOutput($testData, $suiteData) + { + // clear test object handler value to inject parsed content + $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler'); + $property->setAccessible(true); + $property->setValue(null); + + // clear suite object handler value to inject parsed content + $property = new \ReflectionProperty(SuiteObjectHandler::class, 'SUITE_OBJECT_HANLDER_INSTANCE'); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $testData])->make(); + $mockSuiteDataParser = AspectMock::double(SuiteDataParser::class, ['readSuiteData' => $suiteData])->make(); + $instance = AspectMock::double( + ObjectManager::class, + ['create' => function ($clazz) use ($mockDataParser, $mockSuiteDataParser) { + if ($clazz == TestDataParser::class) { + return $mockDataParser; + } + + if ($clazz == SuiteDataParser::class) { + return $mockSuiteDataParser; + } + }] + )->make(); + // bypass the private constructor + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php new file mode 100644 index 000000000..444749ed2 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php @@ -0,0 +1,122 @@ + 100, + 'test2' => 300, + 'test3' => 50, + 'test4' => 60, + 'test5' => 25, + 'test6' => 125, + 'test7' => 250, + 'test8' => 1, + 'test9' => 80, + 'test10' => 25 + ]; + + $expectedResult = [ + 1 => ['test2'], + 2 => ['test7'], + 3 => ['test6', 'test9'], + 4 => ['test1', 'test4', 'test3'], + 5 => ['test5', 'test10', 'test8'] + ]; + + $testSorter = new ParallelGroupSorter(); + $actualResult = $testSorter->getTestsGroupedBySize([], $sampleTestArray, 200); + + $this->assertCount(5, $actualResult); + + foreach ($actualResult as $gropuNumber => $actualTests) { + $expectedTests = $expectedResult[$gropuNumber]; + $this->assertEquals($expectedTests, array_keys($actualTests)); + } + } + + /** + * Test a sort of both tests and a suite which is larger than the given line limitation + */ + public function testSortWithSuites() + { + // mock tests for test object handler. + $numberOfCalls = 0; + $mockTest1 = AspectMock::double(TestObject::class, ['getTestActionCount' => function () use (&$numberOfCalls) { + $actionCount = [200, 275]; + $result = $actionCount[$numberOfCalls]; + $numberOfCalls++; + + return $result; + }])->make(); + + $mockHandler = AspectMock::double( + TestObjectHandler::class, + ['getObject' => function () use ($mockTest1) { + return $mockTest1; + }] + )->make(); + + AspectMock::double(TestObjectHandler::class, ['getInstance' => $mockHandler])->make(); + + // mock a suite object + $mockSuite = AspectMock::double(SuiteObject::class, [ + 'getBeforeHook' => null, + 'getAfterHook' => null, + 'getName' => 'mockSuite1' + ])->make(); + $mockSuiteHandler = AspectMock::double(SuiteObjectHandler::class, ['getObject' => $mockSuite])->make(); + AspectMock::double(SuiteObjectHandler::class, ['getInstance' => $mockSuiteHandler]); + + // create test to size array + $sampleTestArray = [ + 'test1' => 100, + 'test2' => 300, + 'test3' => 500, + 'test4' => 60, + 'test5' => 125 + ]; + + // create mock suite references + $sampleSuiteArray = [ + 'mockTest1' => ['mockSuite1'], + 'mockTest2' => ['mockSuite1'] + ]; + + // perform sort + $testSorter = new ParallelGroupSorter(); + $actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 200); + + // verify the resulting groups + $this->assertCount(5, $actualResult); + + $expectedResults = [ + 1 => ['test3'], + 2 => ['test2'], + 3 => ['mockSuite1_0'], + 4 => ['mockSuite1_1'], + 5 => ['test5', 'test4', 'test1'] + ]; + + foreach ($actualResult as $groupNum => $group) { + $this->assertEquals($expectedResults[$groupNum], array_keys($group)); + } + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/TestNameValidationUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Validation/NameValidationUtilTest.php similarity index 90% rename from dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/TestNameValidationUtilTest.php rename to dev/tests/unit/Magento/FunctionalTestFramework/Util/Validation/NameValidationUtilTest.php index 8b50062ed..595033012 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/TestNameValidationUtilTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Validation/NameValidationUtilTest.php @@ -7,10 +7,10 @@ namespace tests\unit\Magento\FunctionalTestFramework\Test\Util; use Magento\FunctionalTestingFramework\Exceptions\XmlException; -use Magento\FunctionalTestingFramework\Test\Util\TestNameValidationUtil; +use Magento\FunctionalTestingFramework\Util\Validation\NameValidationUtil; use PHPUnit\Framework\TestCase; -class TestNameValidationUtilTest extends TestCase +class NameValidationUtilTest extends TestCase { /** * Validate name with curly braces throws exception @@ -69,6 +69,6 @@ public function testSpacesInTestName() private function validateBlacklistedTestName($testName) { $this->expectException(XmlException::class); - TestNameValidationUtil::validateName($testName); + NameValidationUtil::validateName($testName, "Test"); } } diff --git a/dev/tests/unit/Util/SuiteDataArrayBuilder.php b/dev/tests/unit/Util/SuiteDataArrayBuilder.php new file mode 100644 index 000000000..85411b447 --- /dev/null +++ b/dev/tests/unit/Util/SuiteDataArrayBuilder.php @@ -0,0 +1,234 @@ +name = $name; + return $this; + } + + /** + * Function which takes an array of test names and formats them as included raw suite data + * + * @param array $tests + * @return $this + */ + public function includeTests($tests) + { + $this->includes = $this->appendEntriesToSuiteContents($this->includes, 'test', $tests); + return $this; + } + + /** + * Function which takes an array of test names and formats them as excluded raw suite data + * + * @param array $tests + * @return $this + */ + public function excludeTests($tests) + { + $this->excludes = $this->appendEntriesToSuiteContents($this->excludes, 'test', $tests); + return $this; + } + + /** + * Function which takes an array of group names and formats them as included raw suite data + * + * @param array $groups + * @return $this + */ + public function includeGroups($groups) + { + $this->includes = $this->appendEntriesToSuiteContents($this->includes, 'group', $groups); + return $this; + } + + /** + * Function which takes an array of group names and formats them as excluded raw suite data + * + * @param array $groups + * @return $this + */ + public function excludeGroups($groups) + { + $this->excludes = $this->appendEntriesToSuiteContents($this->excludes, 'groups', $groups); + return $this; + } + + /** + * Function which takes an array of module names and formats them as included raw suite data + * + * @param array $modules + * @return $this + */ + public function includeModules($modules) + { + $this->includes = $this->appendEntriesToSuiteContents($this->includes, 'module', $modules); + return $this; + } + + /** + * Function which takes an array of group names and formats them as excluded raw suite data + * + * @param array $modules + * @return $this + */ + public function excludeModules($modules) + { + $this->excludes = $this->appendEntriesToSuiteContents($this->excludes, 'module', $modules); + return $this; + } + + /** + * Function which takes an array of current include/exclude contents, a type (group, module, or test) and contents + * to be appended to the array and returns a propelry formatted array representative of parsed suite data. + * + * @param array $currentContents + * @param string $type + * @param array $contents + * @return array + */ + private function appendEntriesToSuiteContents($currentContents, $type, $contents) + { + $newContents = $currentContents; + foreach ($contents as $entry) { + $newContents[$entry] = [ + SuiteObjectExtractor::NODE_NAME => $type, + SuiteObjectExtractor::NAME => $entry + ]; + } + + return $newContents; + } + + /** + * Add an after hook passed in by arg (or default if no arg) + * + * @param null $afterHook + * @return $this + */ + public function withAfterHook($afterHook = null) + { + if ($afterHook == null) { + $this->afterHook = [$this->testActionAfterName => [ + ActionObjectExtractor::NODE_NAME => $this->testActionType, + ActionObjectExtractor::TEST_STEP_MERGE_KEY => $this->testActionAfterName + + ]]; + } else { + $this->afterHook = $afterHook; + } + + return $this; + } + + /** + * Add a before hook passed in by arg (or default if no arg) + * + * @param null $beforeHook + * @return $this + */ + public function withBeforeHook($beforeHook = null) + { + if ($beforeHook == null) { + $this->beforeHook = [$this->testActionBeforeName => [ + ActionObjectExtractor::NODE_NAME => $this->testActionType, + ActionObjectExtractor::TEST_STEP_MERGE_KEY => $this->testActionBeforeName + ]]; + } else { + $this->beforeHook = $beforeHook; + } + + return $this; + } + + /** + * Function which takes all class properties set and generates an array representing suite data as parsed from xml. + * + * @return array + */ + public function build() + { + return ['suites' => [ + $this->name => [ + SuiteObjectExtractor::NAME => $this->name, + TestObjectExtractor::TEST_BEFORE_HOOK => $this->beforeHook, + TestObjectExtractor::TEST_AFTER_HOOK => $this->afterHook, + SuiteObjectExtractor::INCLUDE_TAG_NAME => $this->includes, + SuiteObjectExtractor::EXCLUDE_TAG_NAME => $this->excludes + ] + ]]; + } +} diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php index 10049484e..b1790ac81 100644 --- a/dev/tests/verification/Tests/SuiteGenerationTest.php +++ b/dev/tests/verification/Tests/SuiteGenerationTest.php @@ -86,6 +86,7 @@ public function testSuiteGeneration1() // Validate tests have been generated $dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']); + foreach ($expectedContents as $expectedFile) { $this->assertTrue(in_array($expectedFile, $dirContents)); } diff --git a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php index 46c2caaed..5ba1c3cac 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php @@ -40,6 +40,16 @@ class GroupClassGenerator */ private $mustacheEngine; + /** + * Static function to return group directory path for precondition files. + * + * @return string + */ + public static function getGroupDirPath() + { + return dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR . self::GROUP_DIR_NAME . DIRECTORY_SEPARATOR; + } + /** * GroupClassGenerator constructor */ @@ -64,10 +74,7 @@ public function generateGroupClass($suiteObject) { $classContent = $this->createClassContent($suiteObject); $configEntry = self::GROUP_DIR_NAME . DIRECTORY_SEPARATOR . $suiteObject->getName(); - $filePath = dirname(dirname(__DIR__)) . - DIRECTORY_SEPARATOR . - $configEntry . - '.php'; + $filePath = self::getGroupDirPath() . $suiteObject->getName() . '.php'; file_put_contents($filePath, $classContent); return str_replace(DIRECTORY_SEPARATOR, "\\", $configEntry); @@ -122,6 +129,7 @@ private function buildHookMustacheArray($hookObj) $mustacheHookArray = []; $actions = []; $hasWebDriverActions = false; + foreach ($hookObj->getActions() as $action) { /** @var ActionObject $action */ $index = count($actions); diff --git a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php index 6795bef5b..3e4e0a1b5 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php @@ -5,16 +5,12 @@ */ namespace Magento\FunctionalTestingFramework\Suite\Handlers; -use Exception; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; use Magento\FunctionalTestingFramework\Suite\Parsers\SuiteDataParser; use Magento\FunctionalTestingFramework\Suite\Util\SuiteObjectExtractor; -use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; -use Magento\FunctionalTestingFramework\Test\Util\TestHookObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\ObjectExtractor; -use Magento\Ui\Test\Unit\Component\PagingTest; /** * Class SuiteObjectHandler diff --git a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php index c0a2be9e9..bf560eac3 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php @@ -117,6 +117,16 @@ public function requiresGroupFile() return !empty($this->hooks); } + /** + * Getter for the Hook Array which contains the before/after objects. + * + * @return TestHookObject[] + */ + public function getHooks() + { + return $this->hooks; + } + /** * Getter for before hooks. * @@ -124,7 +134,7 @@ public function requiresGroupFile() */ public function getBeforeHook() { - return $this->hooks['before']; + return $this->hooks['before'] ?? null; } /** @@ -134,6 +144,6 @@ public function getBeforeHook() */ public function getAfterHook() { - return $this->hooks['after']; + return $this->hooks['after'] ?? null; } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index a3c664fb0..f87ab5c4c 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -12,6 +12,8 @@ use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest; +use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Yaml\Yaml; @@ -32,13 +34,6 @@ class SuiteGenerator */ private static $SUITE_GENERATOR_INSTANCE; - /** - * Boolean to track whether we have already cleared the yaml file. - * - * @var bool - */ - private $ymlFileCleared = false; - /** * Group Class Generator initialized in constructor. * @@ -62,6 +57,9 @@ private function __construct() public static function getInstance() { if (!self::$SUITE_GENERATOR_INSTANCE) { + // clear any previous configurations before any generation occurs. + self::clearPreviousGroupPreconditions(); + self::clearPreviousSessionConfigEntries(); self::$SUITE_GENERATOR_INSTANCE = new SuiteGenerator(); } @@ -72,17 +70,38 @@ public static function getInstance() * Function which takes all suite configurations and generates to appropriate directory, updating yml configuration * as needed. Returns an array of all tests generated keyed by test name. * - * @param string $config + * @param BaseTestManifest $testManifest + * @return void + */ + public function generateAllSuites($testManifest) + { + $suites = SuiteObjectHandler::getInstance()->getAllObjects(); + if (get_class($testManifest) == ParallelTestManifest::class) { + /** @var ParallelTestManifest $testManifest */ + $suites = $testManifest->getSorter()->getResultingSuites(); + } + + foreach ($suites as $suite) { + // during a parallel config run we must generate only after we have data around how a suite will be split + $this->generateSuiteFromObject($suite); + } + } + + /** + * Returns an array of tests contained within suites as keys pointed at the name of their corresponding suite. + * * @return array */ - public function generateAllSuites($config) + public function getTestsReferencedInSuites() { $testsReferencedInSuites = []; $suites = SuiteObjectHandler::getInstance()->getAllObjects(); foreach ($suites as $suite) { /** @var SuiteObject $suite */ - $testsReferencedInSuites = array_merge($testsReferencedInSuites, $suite->getTests()); - $this->generateSuite($suite->getName(), $config); + $test_keys = array_keys($suite->getTests()); + $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]); + + $testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName); } return $testsReferencedInSuites; @@ -93,23 +112,34 @@ public function generateAllSuites($config) * yml configuration for group run. * * @param string $suiteName - * @param string $config * @return void */ - public function generateSuite($suiteName, $config = null) + public function generateSuite($suiteName) { /**@var SuiteObject $suite **/ $suite = SuiteObjectHandler::getInstance()->getObject($suiteName); + $this->generateSuiteFromObject($suite); + } + + /** + * Function which takes a suite object and generates all relevant supporting files and classes. + * + * @param SuiteObject $suiteObject + * @return void + */ + public function generateSuiteFromObject($suiteObject) + { + $suiteName = $suiteObject->getName(); $relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName; $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath; $groupNamespace = null; DirSetupUtil::createGroupDir($fullPath); - $this->generateRelevantGroupTests($suiteName, $suite->getTests(), $config); + $this->generateRelevantGroupTests($suiteName, $suiteObject->getTests()); - if ($suite->requiresGroupFile()) { + if ($suiteObject->requiresGroupFile()) { // if the suite requires a group file, generate it and set the namespace - $groupNamespace = $this->groupClassGenerator->generateGroupClass($suite); + $groupNamespace = $this->groupClassGenerator->generateGroupClass($suiteObject); } $this->appendEntriesToConfig($suiteName, $fullPath, $groupNamespace); @@ -128,50 +158,33 @@ public function generateSuite($suiteName, $config = null) */ private function appendEntriesToConfig($suiteName, $suitePath, $groupNamespace) { - $configYmlPath = dirname(dirname(TESTS_BP)) . DIRECTORY_SEPARATOR; - $configYmlFile = $configYmlPath . self::YAML_CODECEPTION_CONFIG_FILENAME; - $defaultConfigYmlFile = $configYmlPath . self::YAML_CODECEPTION_DIST_FILENAME; $relativeSuitePath = substr($suitePath, strlen(dirname(dirname(TESTS_BP))) + 1); - $ymlContents = null; - if (file_exists($configYmlFile)) { - $ymlContents = file_get_contents($configYmlFile); - } else { - $ymlContents = file_get_contents($defaultConfigYmlFile); - } - - $ymlArray = Yaml::parse($ymlContents) ?? []; + $ymlArray = self::getYamlFileContents(); if (!array_key_exists(self::YAML_GROUPS_TAG, $ymlArray)) { $ymlArray[self::YAML_GROUPS_TAG]= []; } - $ymlArray = $this->clearPreviousSessionConfigEntries($ymlArray); - if ($groupNamespace) { $ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG][] = $groupNamespace; } $ymlArray[self::YAML_GROUPS_TAG][$suiteName] = [$relativeSuitePath]; $ymlText = self::YAML_COPYRIGHT_TEXT . Yaml::dump($ymlArray, 10); - file_put_contents($configYmlFile, $ymlText); + file_put_contents(self::getYamlConfigFilePath() . self::YAML_CODECEPTION_CONFIG_FILENAME, $ymlText); } /** * Function which takes the current config.yml array and clears any previous configuration for suite group object * files. * - * @param array $ymlArray - * @return array + * @return void */ - private function clearPreviousSessionConfigEntries($ymlArray) + private static function clearPreviousSessionConfigEntries() { - if ($this->ymlFileCleared) { - return $ymlArray; - } - + $ymlArray = self::getYamlFileContents(); $newYmlArray = $ymlArray; // if the yaml entries haven't already been cleared - if (array_key_exists(self::YAML_EXTENSIONS_TAG, $ymlArray)) { foreach ($ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG] as $key => $entry) { if (preg_match('/(Group\\\\.*)/', $entry)) { @@ -190,9 +203,8 @@ private function clearPreviousSessionConfigEntries($ymlArray) unset($newYmlArray[self::YAML_GROUPS_TAG]); } - $this->ymlFileCleared = true; - - return $newYmlArray; + $ymlText = self::YAML_COPYRIGHT_TEXT . Yaml::dump($newYmlArray, 10); + file_put_contents(self::getYamlConfigFilePath() . self::YAML_CODECEPTION_CONFIG_FILENAME, $ymlText); } /** @@ -202,12 +214,52 @@ private function clearPreviousSessionConfigEntries($ymlArray) * * @param string $path * @param array $tests - * @param string $config * @return void */ - private function generateRelevantGroupTests($path, $tests, $config) + private function generateRelevantGroupTests($path, $tests) { $testGenerator = TestGenerator::getInstance($path, $tests); - $testGenerator->createAllTestFiles($config); + $testGenerator->createAllTestFiles('suite'); + } + + /** + * Function which on first execution deletes all generate php in the MFTF Group directory + * + * @return void + */ + private static function clearPreviousGroupPreconditions() + { + $groupFilePath = GroupClassGenerator::getGroupDirPath(); + array_map('unlink', glob("$groupFilePath*.php")); + } + + /** + * Function to return contents of codeception.yml file for config changes. + * + * @return array + */ + private static function getYamlFileContents() + { + $configYmlFile = self::getYamlConfigFilePath() . self::YAML_CODECEPTION_CONFIG_FILENAME; + $defaultConfigYmlFile = self::getYamlConfigFilePath() . self::YAML_CODECEPTION_DIST_FILENAME; + + $ymlContents = null; + if (file_exists($configYmlFile)) { + $ymlContents = file_get_contents($configYmlFile); + } else { + $ymlContents = file_get_contents($defaultConfigYmlFile); + } + + return Yaml::parse($ymlContents) ?? []; + } + + /** + * Static getter for the Config yml filepath (as path cannot be stored in a const) + * + * @return string + */ + private static function getYamlConfigFilePath() + { + return dirname(dirname(TESTS_BP)) . DIRECTORY_SEPARATOR; } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php index 56e50982a..4ff0b25a3 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php @@ -13,6 +13,7 @@ use Magento\FunctionalTestingFramework\Test\Util\BaseObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestHookObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; +use Magento\FunctionalTestingFramework\Util\Validation\NameValidationUtil; class SuiteObjectExtractor extends BaseObjectExtractor { @@ -55,6 +56,9 @@ public function parseSuiteDataIntoObjects($parsedSuiteData) // skip non array items parsed from suite (suite objects will always be arrays) continue; } + + // validate the name used isn't using special char or the "default" reserved name + NameValidationUtil::validateName($parsedSuite[self::NAME], 'Suite'); if ($parsedSuite[self::NAME] == 'default') { throw new XmlException("A Suite can not have the name \"default\""); } diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestNameValidationUtil.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestNameValidationUtil.php deleted file mode 100644 index 61124a3b3..000000000 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestNameValidationUtil.php +++ /dev/null @@ -1,61 +0,0 @@ - "spaces", - "," => "commas", - "'" => "single quotes", - "\"" => "double quotes", - "{" => "curly braces", - "}" => "curly braces", - "$" => "dollar signs", - "(" => "parenthesis", - ")" => "parenthesis" - ]; - - /** - * Function which runs a validation against the blacklisted char defined in this class. Validation occurs to insure - * allure report does not error/future devOps builds do not error against illegal char. - * - * @param string $testName - * @return void - * @throws XmlException - */ - public static function validateName($testName) - { - $testChars = str_split($testName); - - $diff = array_intersect($testChars, array_keys(self::BLACKLISTED_CHAR)); - if (count($diff) > 0) { - $errorMessage = "Test name \"${testName}\" contains illegal characters, please fix and re-run."; - $uniqueDiff = array_unique(array_map(['self', 'nameMapper'], $diff)); - - foreach ($uniqueDiff as $diffChar) { - $errorMessage .= "\nTest names cannot contain " . $diffChar; - } - - throw new XmlException($errorMessage); - } - } - - /** - * Function which maps the blacklisted char to its name, function is used by the array map above. - * - * @param string $val - * @return string - * @SuppressWarnings(PHPMD.UnusedPrivateMethod) - */ - private static function nameMapper($val) - { - return self::BLACKLISTED_CHAR[$val]; - } -} diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php index d9044a0a1..48b7d5552 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php @@ -8,6 +8,7 @@ use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\Test\Objects\TestObject; +use Magento\FunctionalTestingFramework\Util\Validation\NameValidationUtil; /** * Class TestObjectExtractor @@ -61,7 +62,7 @@ public function __construct() public function extractTestData($testData) { // validate the test name for blacklisted char (will cause allure report issues) MQE-483 - TestNameValidationUtil::validateName($testData[self::NAME]); + NameValidationUtil::validateName($testData[self::NAME], "Test"); $testAnnotations = []; $testHooks = []; diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php index f31271a8d..d8c7c397b 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php @@ -57,8 +57,9 @@ abstract public function addTest($testObject); /** * Function which generates the actual manifest(s) once the relevant tests have been added to the array. * + * @param array $testsReferencedInSuites * @param int|null $nodes * @return void */ - abstract public function generate($nodes = null); + abstract public function generate($testsReferencedInSuites, $nodes = null); } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php index 369de08e6..d577be92f 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php @@ -62,10 +62,11 @@ public function addTest($testObject) /** * Function which outputs a list of all test files to the defined testManifest.txt file. * + * @param array $testsReferencedInSuites * @param int|null $nodes * @return void */ - public function generate($nodes = null) + public function generate($testsReferencedInSuites, $nodes = null) { $fileResource = fopen($this->manifestPath, 'a'); @@ -74,9 +75,32 @@ public function generate($nodes = null) fwrite($fileResource, $line . PHP_EOL); } + $this->generateSuiteEntries($testsReferencedInSuites, $fileResource); + fclose($fileResource); } + /** + * Function which takes the test suites passed to the manifest and generates corresponding entries in the manifest. + * + * @param array $testsReferencedInSuites + * @param resource $fileResource + * @return void + */ + protected function generateSuiteEntries($testsReferencedInSuites, $fileResource) + { + // get the names of available suites + $suiteNames = []; + array_walk($testsReferencedInSuites, function ($value) use (&$suiteNames) { + $suiteNames = array_unique(array_merge($value, $suiteNames)); + }); + + foreach ($suiteNames as $suiteName) { + $line = "-g {$suiteName}"; + fwrite($fileResource, $line . PHP_EOL); + } + } + /** * Function which checks the path for an existing test manifest and clears if the file has not already been cleared * during current runtime. diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php index f130ecff7..9c26e270c 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php @@ -7,8 +7,12 @@ namespace Magento\FunctionalTestingFramework\Util\Manifest; use Magento\Framework\Exception\RuntimeException; +use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; +use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; +use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; use Magento\FunctionalTestingFramework\Test\Objects\TestObject; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use Magento\FunctionalTestingFramework\Util\Sorter\ParallelGroupSorter; class ParallelTestManifest extends BaseTestManifest { @@ -21,6 +25,13 @@ class ParallelTestManifest extends BaseTestManifest */ private $testNameToSize = []; + /** + * An instance of the group sorter which will take suites and tests organizing them to be run together. + * + * @var ParallelGroupSorter + */ + private $parallelGroupSorter; + /** * Path to the directory that will contain all test group files * @@ -37,6 +48,7 @@ class ParallelTestManifest extends BaseTestManifest public function __construct($manifestPath, $testPath) { $this->dirPath = $manifestPath . DIRECTORY_SEPARATOR . 'groups'; + $this->parallelGroupSorter = new ParallelGroupSorter(); parent::__construct($testPath, self::PARALLEL_CONFIG); } @@ -54,43 +66,56 @@ public function addTest($testObject) /** * Function which generates the actual manifest once the relevant tests have been added to the array. * - * @param int $nodes + * @param array $testsReferencedInSuites + * @param int $lines * @return void */ - public function generate($nodes = null) + public function generate($testsReferencedInSuites, $lines = null) { - if ($nodes == null) { - $nodes = 2; - } - DirSetupUtil::createGroupDir($this->dirPath); - arsort($this->testNameToSize); - $node = $nodes; + $testGroups = $this->parallelGroupSorter->getTestsGroupedBySize( + $testsReferencedInSuites, + $this->testNameToSize, + $lines + ); - foreach (array_keys($this->testNameToSize) as $testName) { - $node = $this->getNodeOrder($node, $nodes); - $nodeString = strval($node); - $fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeString}.txt", 'a'); - $line = $this->relativeDirPath . DIRECTORY_SEPARATOR . $testName . '.php'; - fwrite($fileResource, $line . PHP_EOL); - fclose($fileResource); + foreach ($testGroups as $groupNumber => $groupContents) { + $this->generateGroupFile($groupContents, $groupNumber); } } /** - * Function which independently iterates node position based on number of desired nodes. + * Function which simply returns the private sorter used by the manifest. * - * @param int $currentNode - * @param int $nodes - * @return int + * @return ParallelGroupSorter */ - private function getNodeOrder($currentNode, $nodes) + public function getSorter() { - $adjustedRef = $currentNode + 1; - if ($adjustedRef <= $nodes) { - return $currentNode + 1; - } + return $this->parallelGroupSorter; + } - return 1; + /** + * Function which takes an array containing entries representing the test execution as well as the associated group + * for the entry in order to generate a txt file used by devops for parllel execution in Jenkins. + * + * @param array $testGroup + * @param int $nodeNumber + * @return void + */ + private function generateGroupFile($testGroup, $nodeNumber) + { + $suites = $this->parallelGroupSorter->getResultingSuites(); + foreach ($testGroup as $entryName => $testValue) { + $fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeNumber}.txt", 'a'); + + $line = null; + if (array_key_exists($entryName, $suites)) { + $line = "-g {$entryName}"; + } else { + $line = $this->relativeDirPath . DIRECTORY_SEPARATOR . $entryName . '.php'; + } + fwrite($fileResource, $line . PHP_EOL); + fclose($fileResource); + } } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php index 5fc492e9e..f4fb33b5e 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php @@ -26,14 +26,16 @@ public function __construct($manifestPath, $testPath) /** * Function which generates the actual manifest once the relevant tests have been added to the array. * + * @param array $testsReferencedInSuites * @param int|null $nodes * @return void */ - public function generate($nodes = null) + public function generate($testsReferencedInSuites, $nodes = null) { $fileResource = fopen($this->manifestPath, 'a'); $line = $this->relativeDirPath . DIRECTORY_SEPARATOR; fwrite($fileResource, $line . PHP_EOL); + $this->generateSuiteEntries($testsReferencedInSuites, $fileResource); fclose($fileResource); } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php index bc54a47ea..4344a96a1 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php @@ -33,6 +33,10 @@ public static function makeManifest($manifestPath, $testPath, $runConfig) case 'parallel': return new ParallelTestManifest($manifestPath, $testPath); + case 'suite': + // the suite does not have its own manifest but instead is handled by the other suite types. + return null; + default: return new DefaultTestManifest($manifestPath, $testPath); diff --git a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php new file mode 100644 index 000000000..9d51bf2e1 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php @@ -0,0 +1,270 @@ +createGroupsWithinSuites($testNameToSuiteName, $lines); + $splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests); + $entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize); + arsort($entriesForGeneration); + + $testNameToSizeForUse = $entriesForGeneration; + $nodeNumber = 1; + foreach ($entriesForGeneration as $testName => $testSize) { + if (!array_key_exists($testName, $testNameToSizeForUse)) { + // skip tests which have already been added to a group + continue; + } + + $testGroup = $this->createTestGroup($lines, $testName, $testSize, $testNameToSizeForUse); + $testGroups[$nodeNumber] = $testGroup; + + // unset the test which have been used. + $testNameToSizeForUse = array_diff($testNameToSizeForUse, $testGroup); + $nodeNumber++; + } + + return $testGroups; + } + + /** + * Function which returns the newly formed suite objects created as a part of the sort + * + * @return array + */ + public function getResultingSuites() + { + return $this->compositeSuiteObjects; + } + + /** + * Function which constructs a group of tests to be run together based on the desired number of lines per group, + * a test to be used as a starting point, the size of a starting test, an array of tests available to be added to + * the group. + * + * @param integer $lineMaximum + * @param string $testName + * @param integer $testSize + * @param array $testNameToSizeForUse + * @return array + */ + private function createTestGroup($lineMaximum, $testName, $testSize, $testNameToSizeForUse) + { + $group[$testName] = $testSize; + + if ($testSize < $lineMaximum) { + while (array_sum($group) < $lineMaximum && !empty($testNameToSizeForUse)) { + $groupSize = array_sum($group); + $lineGoal = $lineMaximum - $groupSize; + + $testNameForUse = $this->getClosestLineCount($testNameToSizeForUse, $lineGoal); + $testSizeForUse = $testNameToSizeForUse[$testNameForUse]; + unset($testNameToSizeForUse[$testNameForUse]); + + $group[$testNameForUse] = $testSizeForUse; + } + } + + return $group; + } + + /** + * Function which takes a group of available tests mapped to size and a desired number of lines matching with the + * test of closest size and returning. + * + * @param array $testGroup + * @param integer $desiredValue + * @return string + */ + private function getClosestLineCount($testGroup, $desiredValue) + { + $winner = key($testGroup); + $closestThreshold = $desiredValue; + foreach ($testGroup as $testName => $testValue) { + $testThreshold = abs($desiredValue - $testValue); + if ($closestThreshold > $testThreshold) { + $closestThreshold = $testThreshold; + $winner = $testName; + } + } + + return $winner; + } + + /** + * Function which takes an array of test names mapped to suite name and a size limitation for each group of tests. + * The function divides suites that are over the specified limit and returns the resulting suites in an array. + * + * @param array $testNameToSuiteName + * @param integer $lineLimit + * @return array + */ + private function createGroupsWithinSuites($testNameToSuiteName, $lineLimit) + { + $suiteNameToTestNames = []; + $suiteNameToSize = []; + array_walk($testNameToSuiteName, function ($value, $key) use (&$suiteNameToTestNames, &$suiteNameToSize) { + $testActionCount = TestObjectHandler::getInstance()->getObject($key)->getTestActionCount(); + foreach ($value as $suite) { + $suiteNameToTestNames[$suite][$key] = $testActionCount; + $currentSize = $suiteNameToSize[$suite] ?? 0; + $suiteNameToSize[$suite] = $currentSize + $testActionCount; + } + }); + + // divide the suites up within the array + $suitesForResize = array_filter($suiteNameToSize, function ($val) use ($lineLimit) { + return $val > $lineLimit; + }); + + // remove the suites for resize from the original list + $remainingSuites = array_diff_key($suiteNameToTestNames, $suitesForResize); + + foreach ($remainingSuites as $remainingSuite => $tests) { + $this->addSuiteAsObject($remainingSuite, null, null); + } + + $resultingGroups = []; + foreach ($suitesForResize as $suiteName => $suiteSize) { + $resultingGroups = array_merge( + $resultingGroups, + $this->splitTestSuite($suiteName, $suiteNameToTestNames[$suiteName], $lineLimit) + ); + } + + // merge the resulting divisions with the appropriately sized suites + return array_merge($remainingSuites, $resultingGroups); + } + + /** + * Function which takes a multidimensional array containing a suite name mapped to an array of tests names as keys + * with their sizes as values. The function returns an array of suite name to size of the corresponding mapped + * tests. + * + * @param array $suiteNamesToTests + * @return array + */ + private function getSuiteToSize($suiteNamesToTests) + { + $suiteNamesToSize = []; + foreach ($suiteNamesToTests as $name => $tests) { + $size = array_sum($tests); + $suiteNamesToSize[$name] = $size; + } + + return $suiteNamesToSize; + } + + /** + * Function which takes a suite name, an array of tests affiliated with that suite, and a maximum number of lines. + * The function uses the limit to split up the oversized suite and returns an array of suites representative of the + * previously oversized suite. + * + * E.g. + * Input {suitename = 'sample', tests = ['test1' => 100,'test2' => 150, 'test3' => 300], linelimit = 275} + * Result { ['sample_01' => ['test3' => 300], 'sample_02' => ['test2' => 150, 'test1' => 100]] } + * + * @param string $suiteName + * @param array $tests + * @param integer $lineLimit + * @return array + */ + private function splitTestSuite($suiteName, $tests, $lineLimit) + { + arsort($tests); + $split_suites = []; + $availableTests = $tests; + $split_count = 0; + + foreach ($tests as $test => $size) { + if (!array_key_exists($test, $availableTests)) { + continue; + } + + $group = $this->createTestGroup($lineLimit, $test, $size, $availableTests); + $split_suites["{$suiteName}_${split_count}"] = $group; + $this->addSuiteAsObject($suiteName, "{$suiteName}_${split_count}", $group); + + $availableTests = array_diff($availableTests, $group); + $split_count++; + } + + return $split_suites; + } + + /** + * Function which takes a new suite, the original suite from which it was sourced, and an array of tests now + * associated with thew new suite. The function takes this information and creates a new suite object stored in + * the sorter for later retrieval, copying the pre/post conditions from the original suite. + * + * @param string $originalSuiteName + * @param string $newSuiteName + * @param array $tests + * @return void + */ + private function addSuiteAsObject($originalSuiteName, $newSuiteName, $tests) + { + /** @var SuiteObject $originalSuite */ + $originalSuite = SuiteObjectHandler::getInstance()->getObject($originalSuiteName); + if ($newSuiteName == null && $tests == null) { + $this->compositeSuiteObjects[$originalSuiteName] = $originalSuite; + return; + } + + $newSuiteTests = []; + foreach ($tests as $test => $lines) { + $newSuiteTests[$test] = TestObjectHandler::getInstance()->getObject($test); + } + + $this->compositeSuiteObjects[$newSuiteName] = new SuiteObject( + $newSuiteName, + $newSuiteTests, + [], + $originalSuite->getHooks() + ); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 4a51906c0..3a3483ea4 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -154,7 +154,7 @@ private function createCestFile($testPhp, $filename) * @param string $runConfig * @param int $nodes * @param TestObject[] $testsToIgnore - * @return void + * @return BaseTestManifest * @throws TestReferenceException * @throws \Exception */ @@ -173,6 +173,8 @@ public function createAllTestFiles($runConfig = null, $nodes = null, $testsToIgn foreach ($testPhpArray as $testPhpFile) { $this->createCestFile($testPhpFile[1], $testPhpFile[0]); } + + return $testManifest; } /** @@ -230,15 +232,21 @@ private function assembleAllTestPhp($testManifest, $nodes, array $testsToIgnore) $php = $this->assembleTestPhp($test); $cestPhpArray[] = [$test->getCodeceptionName(), $php]; - //write to manifest here if config is not single run - $testManifest->addTest($test); + $debugInformation = $test->getDebugInformation(); $this->debug($debugInformation); $this->debug("Finish creating test: " . $test->getCodeceptionName() . "" . PHP_EOL); + + //write to manifest here if manifest is not null + if ($testManifest != null) { + $testManifest->addTest($test); + } } - $testManifest->generate($nodes); + if ($testManifest != null) { + $testManifest->generate($testsToIgnore, intval($nodes)); + } return $cestPhpArray; } diff --git a/src/Magento/FunctionalTestingFramework/Util/Validation/NameValidationUtil.php b/src/Magento/FunctionalTestingFramework/Util/Validation/NameValidationUtil.php new file mode 100644 index 000000000..60b4660fb --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Util/Validation/NameValidationUtil.php @@ -0,0 +1,53 @@ + Date: Wed, 4 Apr 2018 16:04:54 -0500 Subject: [PATCH 2/2] MQE-878: Modify parallel grouping algorithm to work with suites - fix for static check post merge --- src/Magento/FunctionalTestingFramework/Util/TestGenerator.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 3a3483ea4..a18d1e086 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -232,9 +232,7 @@ private function assembleAllTestPhp($testManifest, $nodes, array $testsToIgnore) $php = $this->assembleTestPhp($test); $cestPhpArray[] = [$test->getCodeceptionName(), $php]; - $debugInformation = $test->getDebugInformation(); - $this->debug($debugInformation); $this->debug("Finish creating test: " . $test->getCodeceptionName() . "" . PHP_EOL);