diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php index 6c6efe8ea..c4a04d361 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php @@ -36,9 +36,9 @@ public function testBasicTestGroupSort() $expectedResult = [ 1 => ['test2'], 2 => ['test7'], - 3 => ['test6', 'test9'], - 4 => ['test1', 'test4', 'test3'], - 5 => ['test5', 'test10', 'test8'] + 3 => ['test6', 'test4', 'test8'], + 4 => ['test1', 'test9'], + 5 => ['test3', 'test5', 'test10'] ]; $testSorter = new ParallelGroupSorter(); @@ -59,13 +59,16 @@ public function testSortWithSuites() { // mock tests for test object handler. $numberOfCalls = 0; - $mockTest1 = AspectMock::double(TestObject::class, ['getTestActionCount' => function () use (&$numberOfCalls) { - $actionCount = [200, 275]; - $result = $actionCount[$numberOfCalls]; - $numberOfCalls++; - - return $result; - }])->make(); + $mockTest1 = AspectMock::double( + TestObject::class, + ['getEstimatedDuration' => function () use (&$numberOfCalls) { + $actionCount = [300, 275]; + $result = $actionCount[$numberOfCalls]; + $numberOfCalls++; + + return $result; + }] + )->make(); $mockHandler = AspectMock::double( TestObjectHandler::class, @@ -92,17 +95,16 @@ public function testSortWithSuites() // perform sort $testSorter = new ParallelGroupSorter(); - $actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 200); + $actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 500); // verify the resulting groups - $this->assertCount(5, $actualResult); + $this->assertCount(4, $actualResult); $expectedResults = [ 1 => ['test3'], - 2 => ['test2'], - 3 => ['mockSuite1_0'], - 4 => ['mockSuite1_1'], - 5 => ['test5', 'test4', 'test1'] + 2 => ['test2','test5', 'test4'], + 3 => ['mockSuite1_0', 'test1'], + 4 => ['mockSuite1_1'] ]; foreach ($actualResult as $groupNum => $group) { diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index a8d34f1c9..119fe48f9 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -35,7 +35,7 @@ protected function configure() ->addArgument('name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'name(s) of specific tests to generate') ->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default') ->addOption("force", 'f',InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration') - ->addOption('lines', 'l', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size', 500) + ->addOption('time', 'i', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size (in minutes)', 10) ->addOption('tests', 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration'); } @@ -45,7 +45,7 @@ protected function configure() * @param InputInterface $input * @param OutputInterface $output * @return void - * @throws \Symfony\Component\Console\Exception\LogicException + * @throws \Symfony\Component\Console\Exception\LogicException|TestFrameworkException */ protected function execute(InputInterface $input, OutputInterface $output) { @@ -53,7 +53,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $config = $input->getOption('config'); $json = $input->getOption('tests'); $force = $input->getOption('force'); - $lines = $input->getOption('lines'); + $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds $verbose = $output->isVerbose(); if ($json !== null && !json_decode($json)) { @@ -61,6 +61,11 @@ protected function execute(InputInterface $input, OutputInterface $output) throw new TestFrameworkException("JSON could not be parsed: " . json_last_error_msg()); } + if ($config === 'parallel' && $time <= 0) { + // stop execution if the user has given us an invalid argument for time argument during parallel generation + throw new TestFrameworkException("time option cannot be less than or equal to 0"); + } + $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $verbose); // create our manifest file here @@ -69,13 +74,13 @@ protected function execute(InputInterface $input, OutputInterface $output) if ($config == 'parallel') { /** @var ParallelTestManifest $testManifest */ - $testManifest->createTestGroups($lines); + $testManifest->createTestGroups($time); } SuiteGenerator::getInstance()->generateAllSuites($testManifest); $testManifest->generate(); - print "Generate Tests Command Run" . PHP_EOL; + $output->writeln("Generate Tests Command Run"); } /** diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php index 7a316fc3c..3569b189a 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php @@ -15,6 +15,18 @@ */ class TestObject { + const WAIT_TIME_ATTRIBUTE = 'time'; + + const TEST_ACTION_WEIGHT = [ + 'waitForPageLoad' => 1500, + 'amOnPage' => 1000, + 'waitForLoadingMaskToDisappear' => 500, + 'wait' => self::WAIT_TIME_ATTRIBUTE, + 'comment' => 5, + 'assertCount' => 5, + 'closeAdminNotification' => 10 + ]; + /** * Name of the test * @@ -159,28 +171,57 @@ public function getHooks() } /** - * Returns the number of a test actions contained within a single test (including before/after actions). + * Returns the estimated duration of a single test (including before/after actions). * * @return int */ - public function getTestActionCount() + public function getEstimatedDuration() { // a skipped action results in a single skip being appended to the beginning of the test and no execution if ($this->isSkipped()) { return 1; } - $hookActions = 0; + $hookTime = 0; if (array_key_exists('before', $this->hooks)) { - $hookActions += count($this->hooks['before']->getActions()); + $hookTime += $this->calculateWeightedActionTimes($this->hooks['before']->getActions()); } if (array_key_exists('after', $this->hooks)) { - $hookActions += count($this->hooks['after']->getActions()); + $hookTime += $this->calculateWeightedActionTimes($this->hooks['after']->getActions()); + } + + $testTime = $this->calculateWeightedActionTimes($this->getOrderedActions()); + + return $hookTime + $testTime; + } + + /** + * Function which takes a set of actions and estimates time for completion based on action type. + * + * @param ActionObject[] $actions + * @return int + */ + private function calculateWeightedActionTimes($actions) + { + $actionTime = 0; + // search for any actions of special type + foreach ($actions as $action) { + /** @var ActionObject $action */ + if (array_key_exists($action->getType(), self::TEST_ACTION_WEIGHT)) { + $weight = self::TEST_ACTION_WEIGHT[$action->getType()]; + if ($weight === self::WAIT_TIME_ATTRIBUTE) { + $weight = intval($action->getCustomActionAttributes()[$weight]) * 1000; + } + + $actionTime += $weight; + continue; + } + + $actionTime += 50; } - $testActions = count($this->getOrderedActions()); - return $hookActions + $testActions; + return $actionTime; } /** diff --git a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php index 53871a098..248e11288 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php +++ b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php @@ -77,7 +77,7 @@ private function parseEnvFile(): array $envContents = $this->parseEnvFileLines($envFile); } - return array_diff_key($this->parseEnvFileLines($envExampleFile), $envContents); + return array_merge($this->parseEnvFileLines($envExampleFile), $envContents); } private function parseEnvFileLines(array $file): array diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php index 12ce5d701..c60634bd3 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php @@ -70,22 +70,22 @@ public function __construct($suiteConfiguration, $testPath) */ public function addTest($testObject) { - $this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getTestActionCount(); + $this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getEstimatedDuration(); } /** * Function which generates test groups based on arg passed. The function builds groups using the args as an upper * limit. * - * @param int $lines + * @param int $time * @return void */ - public function createTestGroups($lines) + public function createTestGroups($time) { $this->testGroups = $this->parallelGroupSorter->getTestsGroupedBySize( $this->getSuiteConfig(), $this->testNameToSize, - $lines + $time ); $this->suiteConfiguration = $this->parallelGroupSorter->getResultingSuiteConfig(); diff --git a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php index 555eb5244..eff9c6d8e 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php +++ b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php @@ -30,22 +30,22 @@ public function __construct() * * @param array $suiteConfiguration * @param array $testNameToSize - * @param integer $lines + * @param integer $time * @return array * @throws TestFrameworkException */ - public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lines) + public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $time) { // we must have the lines argument in order to create the test groups - if ($lines == 0) { + if ($time == 0) { throw new TestFrameworkException( - "Please provide the argument '--lines' to the robo command in order to". + "Please provide the argument '--time' to the robo command in order to". " generate grouped tests manifests for a parallel execution" ); } $testGroups = []; - $splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $lines); + $splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $time); $splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests); $entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize); arsort($entriesForGeneration); @@ -58,7 +58,7 @@ public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lin continue; } - $testGroup = $this->createTestGroup($lines, $testName, $testSize, $testNameToSizeForUse); + $testGroup = $this->createTestGroup($time, $testName, $testSize, $testNameToSizeForUse); $testGroups[$nodeNumber] = $testGroup; // unset the test which have been used. @@ -88,26 +88,29 @@ public function getResultingSuiteConfig() * a test to be used as a starting point, the size of a starting test, an array of tests available to be added to * the group. * - * @param integer $lineMaximum + * @param integer $timeMaximum * @param string $testName * @param integer $testSize * @param array $testNameToSizeForUse * @return array */ - private function createTestGroup($lineMaximum, $testName, $testSize, $testNameToSizeForUse) + private function createTestGroup($timeMaximum, $testName, $testSize, $testNameToSizeForUse) { $group[$testName] = $testSize; - if ($testSize < $lineMaximum) { - while (array_sum($group) < $lineMaximum && !empty($testNameToSizeForUse)) { + if ($testSize < $timeMaximum) { + while (array_sum($group) < $timeMaximum && !empty($testNameToSizeForUse)) { $groupSize = array_sum($group); - $lineGoal = $lineMaximum - $groupSize; + $lineGoal = $timeMaximum - $groupSize; $testNameForUse = $this->getClosestLineCount($testNameToSizeForUse, $lineGoal); - $testSizeForUse = $testNameToSizeForUse[$testNameForUse]; + if ($testNameToSizeForUse[$testNameForUse] < $lineGoal) { + $testSizeForUse = $testNameToSizeForUse[$testNameForUse]; + $group[$testNameForUse] = $testSizeForUse; + } + unset($testNameToSizeForUse[$testNameForUse]); - $group[$testNameForUse] = $testSizeForUse; } } @@ -127,8 +130,12 @@ private function getClosestLineCount($testGroup, $desiredValue) $winner = key($testGroup); $closestThreshold = $desiredValue; foreach ($testGroup as $testName => $testValue) { - $testThreshold = abs($desiredValue - $testValue); - if ($closestThreshold > $testThreshold) { + // find the difference between the desired value and test candidate for the group + $testThreshold = $desiredValue - $testValue; + + // if we see that the gap between the desired value is non-negative and lower than the current closest make + // the test the winner. + if ($closestThreshold > $testThreshold && $testThreshold > 0) { $closestThreshold = $testThreshold; $winner = $testName; } @@ -187,7 +194,7 @@ private function getSuiteNameToTestSize($suiteConfiguration) foreach ($test as $testName) { $suiteNameToTestSize[$suite][$testName] = TestObjectHandler::getInstance() ->getObject($testName) - ->getTestActionCount(); + ->getEstimatedDuration(); } } @@ -224,30 +231,30 @@ private function getSuiteToSize($suiteNamesToTests) * * @param string $suiteName * @param array $tests - * @param integer $lineLimit + * @param integer $maxTime * @return array */ - private function splitTestSuite($suiteName, $tests, $lineLimit) + private function splitTestSuite($suiteName, $tests, $maxTime) { arsort($tests); - $split_suites = []; + $splitSuites = []; $availableTests = $tests; - $split_count = 0; + $splitCount = 0; foreach ($tests as $test => $size) { if (!array_key_exists($test, $availableTests)) { continue; } - $group = $this->createTestGroup($lineLimit, $test, $size, $availableTests); - $split_suites["{$suiteName}_${split_count}"] = $group; - $this->addSuiteToConfig($suiteName, "{$suiteName}_${split_count}", $group); + $group = $this->createTestGroup($maxTime, $test, $size, $availableTests); + $splitSuites["{$suiteName}_${splitCount}"] = $group; + $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group); $availableTests = array_diff_key($availableTests, $group); - $split_count++; + $splitCount++; } - return $split_suites; + return $splitSuites; } /**