Skip to content

Commit 7525790

Browse files
imeron2433imeron2433
committed
MQE-1024: Refactor Algorithm For Mftf Build Groups
- update sorting functions to account for specific action time - update sorting functions to never exceed desired time - update tests following function changes
1 parent 343009b commit 7525790

File tree

6 files changed

+100
-52
lines changed

6 files changed

+100
-52
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ public function testBasicTestGroupSort()
3636
$expectedResult = [
3737
1 => ['test2'],
3838
2 => ['test7'],
39-
3 => ['test6', 'test9'],
40-
4 => ['test1', 'test4', 'test3'],
41-
5 => ['test5', 'test10', 'test8']
39+
3 => ['test6', 'test4', 'test8'],
40+
4 => ['test1', 'test9'],
41+
5 => ['test3', 'test5', 'test10']
4242
];
4343

4444
$testSorter = new ParallelGroupSorter();
@@ -59,13 +59,16 @@ public function testSortWithSuites()
5959
{
6060
// mock tests for test object handler.
6161
$numberOfCalls = 0;
62-
$mockTest1 = AspectMock::double(TestObject::class, ['getTestActionCount' => function () use (&$numberOfCalls) {
63-
$actionCount = [200, 275];
64-
$result = $actionCount[$numberOfCalls];
65-
$numberOfCalls++;
66-
67-
return $result;
68-
}])->make();
62+
$mockTest1 = AspectMock::double(
63+
TestObject::class,
64+
['getEstimatedDuration' => function () use (&$numberOfCalls) {
65+
$actionCount = [300, 275];
66+
$result = $actionCount[$numberOfCalls];
67+
$numberOfCalls++;
68+
69+
return $result;
70+
}]
71+
)->make();
6972

7073
$mockHandler = AspectMock::double(
7174
TestObjectHandler::class,
@@ -92,17 +95,16 @@ public function testSortWithSuites()
9295

9396
// perform sort
9497
$testSorter = new ParallelGroupSorter();
95-
$actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 200);
98+
$actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 500);
9699

97100
// verify the resulting groups
98-
$this->assertCount(5, $actualResult);
101+
$this->assertCount(4, $actualResult);
99102

100103
$expectedResults = [
101104
1 => ['test3'],
102-
2 => ['test2'],
103-
3 => ['mockSuite1_0'],
104-
4 => ['mockSuite1_1'],
105-
5 => ['test5', 'test4', 'test1']
105+
2 => ['test2','test5', 'test4'],
106+
3 => ['mockSuite1_0', 'test1'],
107+
4 => ['mockSuite1_1']
106108
];
107109

108110
foreach ($actualResult as $groupNum => $group) {

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ protected function configure()
3535
->addArgument('name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'name(s) of specific tests to generate')
3636
->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
3737
->addOption("force", 'f',InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration')
38-
->addOption('lines', 'l', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size', 500)
38+
->addOption('time', 'i', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size (in minutes)', 10)
3939
->addOption('tests', 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration');
4040
}
4141

@@ -45,15 +45,15 @@ protected function configure()
4545
* @param InputInterface $input
4646
* @param OutputInterface $output
4747
* @return void
48-
* @throws \Symfony\Component\Console\Exception\LogicException
48+
* @throws \Symfony\Component\Console\Exception\LogicException|TestFrameworkException
4949
*/
5050
protected function execute(InputInterface $input, OutputInterface $output)
5151
{
5252
$tests = $input->getArgument('name');
5353
$config = $input->getOption('config');
5454
$json = $input->getOption('tests');
5555
$force = $input->getOption('force');
56-
$lines = $input->getOption('lines');
56+
$time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
5757
$verbose = $output->isVerbose();
5858

5959
if ($json !== null && !json_decode($json)) {
@@ -69,13 +69,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
6969

7070
if ($config == 'parallel') {
7171
/** @var ParallelTestManifest $testManifest */
72-
$testManifest->createTestGroups($lines);
72+
$testManifest->createTestGroups($time);
7373
}
7474

7575
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
7676
$testManifest->generate();
7777

78-
print "Generate Tests Command Run" . PHP_EOL;
78+
$output->writeln("Generate Tests Command Run");
7979
}
8080

8181
/**

‎src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,16 @@
1515
*/
1616
class TestObject
1717
{
18+
const TEST_ACTION_WEIGHT = [
19+
'waitForPageLoad' => 1500,
20+
'amOnPage' => 1000,
21+
'waitForLoadingMaskToDisappear' => 500,
22+
'wait' => 'time',
23+
'comment' => 5,
24+
'assertCount' => 5,
25+
'closeAdminNotification' => 10
26+
];
27+
1828
/**
1929
* Name of the test
2030
*
@@ -159,28 +169,57 @@ public function getHooks()
159169
}
160170

161171
/**
162-
* Returns the number of a test actions contained within a single test (including before/after actions).
172+
* Returns the estimated duration of a single test (including before/after actions).
163173
*
164174
* @return int
165175
*/
166-
public function getTestActionCount()
176+
public function getEstimatedDuration()
167177
{
168178
// a skipped action results in a single skip being appended to the beginning of the test and no execution
169179
if ($this->isSkipped()) {
170180
return 1;
171181
}
172182

173-
$hookActions = 0;
183+
$hookTime = 0;
174184
if (array_key_exists('before', $this->hooks)) {
175-
$hookActions += count($this->hooks['before']->getActions());
185+
$hookTime += $this->calculateWeightedActionTimes($this->hooks['before']->getActions());
176186
}
177187

178188
if (array_key_exists('after', $this->hooks)) {
179-
$hookActions += count($this->hooks['after']->getActions());
189+
$hookTime += $this->calculateWeightedActionTimes($this->hooks['after']->getActions());
190+
}
191+
192+
$testTime = $this->calculateWeightedActionTimes($this->getOrderedActions());
193+
194+
return $hookTime + $testTime;
195+
}
196+
197+
/**
198+
* Function which takes a set of actions and estimates time for completion based on action type.
199+
*
200+
* @param ActionObject[] $actions
201+
* @return int
202+
*/
203+
private function calculateWeightedActionTimes($actions)
204+
{
205+
$actionTime = 0;
206+
// search for any actions of special type
207+
foreach ($actions as $action) {
208+
/** @var ActionObject $action */
209+
if (array_key_exists($action->getType(), self::TEST_ACTION_WEIGHT)) {
210+
$weight = self::TEST_ACTION_WEIGHT[$action->getType()];
211+
if (is_string($weight) && $weight === 'time') {
212+
$weight = intval($action->getCustomActionAttributes()[$weight]) * 1000;
213+
}
214+
215+
$actionTime += $weight;
216+
continue;
217+
}
218+
219+
$actionTime += 50;
180220
}
181221

182-
$testActions = count($this->getOrderedActions());
183-
return $hookActions + $testActions;
222+
return $actionTime;
184223
}
185224

186225
/**

‎src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ private function parseEnvFile(): array
7777
$envContents = $this->parseEnvFileLines($envFile);
7878
}
7979

80-
return array_diff_key($this->parseEnvFileLines($envExampleFile), $envContents);
80+
return array_merge($this->parseEnvFileLines($envExampleFile), $envContents);
8181
}
8282

8383
private function parseEnvFileLines(array $file): array

‎src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,22 +70,22 @@ public function __construct($suiteConfiguration, $testPath)
7070
*/
7171
public function addTest($testObject)
7272
{
73-
$this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getTestActionCount();
73+
$this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getEstimatedDuration();
7474
}
7575

7676
/**
7777
* Function which generates test groups based on arg passed. The function builds groups using the args as an upper
7878
* limit.
7979
*
80-
* @param int $lines
80+
* @param int $time
8181
* @return void
8282
*/
83-
public function createTestGroups($lines)
83+
public function createTestGroups($time)
8484
{
8585
$this->testGroups = $this->parallelGroupSorter->getTestsGroupedBySize(
8686
$this->getSuiteConfig(),
8787
$this->testNameToSize,
88-
$lines
88+
$time
8989
);
9090

9191
$this->suiteConfiguration = $this->parallelGroupSorter->getResultingSuiteConfig();

‎src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,22 @@ public function __construct()
3030
*
3131
* @param array $suiteConfiguration
3232
* @param array $testNameToSize
33-
* @param integer $lines
33+
* @param integer $time
3434
* @return array
3535
* @throws TestFrameworkException
3636
*/
37-
public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lines)
37+
public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $time)
3838
{
3939
// we must have the lines argument in order to create the test groups
40-
if ($lines == 0) {
40+
if ($time == 0) {
4141
throw new TestFrameworkException(
42-
"Please provide the argument '--lines' to the robo command in order to".
42+
"Please provide the argument '--time' to the robo command in order to".
4343
" generate grouped tests manifests for a parallel execution"
4444
);
4545
}
4646

4747
$testGroups = [];
48-
$splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $lines);
48+
$splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $time);
4949
$splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests);
5050
$entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize);
5151
arsort($entriesForGeneration);
@@ -58,7 +58,7 @@ public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lin
5858
continue;
5959
}
6060

61-
$testGroup = $this->createTestGroup($lines, $testName, $testSize, $testNameToSizeForUse);
61+
$testGroup = $this->createTestGroup($time, $testName, $testSize, $testNameToSizeForUse);
6262
$testGroups[$nodeNumber] = $testGroup;
6363

6464
// unset the test which have been used.
@@ -88,26 +88,29 @@ public function getResultingSuiteConfig()
8888
* a test to be used as a starting point, the size of a starting test, an array of tests available to be added to
8989
* the group.
9090
*
91-
* @param integer $lineMaximum
91+
* @param integer $timeMaximum
9292
* @param string $testName
9393
* @param integer $testSize
9494
* @param array $testNameToSizeForUse
9595
* @return array
9696
*/
97-
private function createTestGroup($lineMaximum, $testName, $testSize, $testNameToSizeForUse)
97+
private function createTestGroup($timeMaximum, $testName, $testSize, $testNameToSizeForUse)
9898
{
9999
$group[$testName] = $testSize;
100100

101-
if ($testSize < $lineMaximum) {
102-
while (array_sum($group) < $lineMaximum && !empty($testNameToSizeForUse)) {
101+
if ($testSize < $timeMaximum) {
102+
while (array_sum($group) < $timeMaximum && !empty($testNameToSizeForUse)) {
103103
$groupSize = array_sum($group);
104-
$lineGoal = $lineMaximum - $groupSize;
104+
$lineGoal = $timeMaximum - $groupSize;
105105

106106
$testNameForUse = $this->getClosestLineCount($testNameToSizeForUse, $lineGoal);
107-
$testSizeForUse = $testNameToSizeForUse[$testNameForUse];
107+
if ($testNameToSizeForUse[$testNameForUse] < $lineGoal) {
108+
$testSizeForUse = $testNameToSizeForUse[$testNameForUse];
109+
$group[$testNameForUse] = $testSizeForUse;
110+
}
111+
108112
unset($testNameToSizeForUse[$testNameForUse]);
109113

110-
$group[$testNameForUse] = $testSizeForUse;
111114
}
112115
}
113116

@@ -127,8 +130,12 @@ private function getClosestLineCount($testGroup, $desiredValue)
127130
$winner = key($testGroup);
128131
$closestThreshold = $desiredValue;
129132
foreach ($testGroup as $testName => $testValue) {
130-
$testThreshold = abs($desiredValue - $testValue);
131-
if ($closestThreshold > $testThreshold) {
133+
// find the difference between the desired value and test candidate for the group
134+
$testThreshold = $desiredValue - $testValue;
135+
136+
// if we see that the gap between the desired value is non-negative and lower than the current closest make
137+
// the test the winner.
138+
if ($closestThreshold > $testThreshold && $testThreshold > 0) {
132139
$closestThreshold = $testThreshold;
133140
$winner = $testName;
134141
}
@@ -187,7 +194,7 @@ private function getSuiteNameToTestSize($suiteConfiguration)
187194
foreach ($test as $testName) {
188195
$suiteNameToTestSize[$suite][$testName] = TestObjectHandler::getInstance()
189196
->getObject($testName)
190-
->getTestActionCount();
197+
->getEstimatedDuration();
191198
}
192199
}
193200

@@ -224,10 +231,10 @@ private function getSuiteToSize($suiteNamesToTests)
224231
*
225232
* @param string $suiteName
226233
* @param array $tests
227-
* @param integer $lineLimit
234+
* @param integer $maxTime
228235
* @return array
229236
*/
230-
private function splitTestSuite($suiteName, $tests, $lineLimit)
237+
private function splitTestSuite($suiteName, $tests, $maxTime)
231238
{
232239
arsort($tests);
233240
$split_suites = [];
@@ -239,7 +246,7 @@ private function splitTestSuite($suiteName, $tests, $lineLimit)
239246
continue;
240247
}
241248

242-
$group = $this->createTestGroup($lineLimit, $test, $size, $availableTests);
249+
$group = $this->createTestGroup($maxTime, $test, $size, $availableTests);
243250
$split_suites["{$suiteName}_${split_count}"] = $group;
244251
$this->addSuiteToConfig($suiteName, "{$suiteName}_${split_count}", $group);
245252

0 commit comments

Comments
 (0)