MQE-1024: Refactor Algorithm For Mftf Build Groups

dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

@@ -36,9 +36,9 @@ public function testBasicTestGroupSort()
         $expectedResult = [
             1 => ['test2'],
             2 => ['test7'],
-            3 => ['test6', 'test9'],
-            4 => ['test1', 'test4', 'test3'],
-            5 => ['test5', 'test10', 'test8']
+            3 => ['test6', 'test4', 'test8'],
+            4 => ['test1', 'test9'],
+            5 => ['test3', 'test5', 'test10']
         ];
 
         $testSorter = new ParallelGroupSorter();
@@ -59,13 +59,16 @@ public function testSortWithSuites()
     {
         // mock tests for test object handler.
         $numberOfCalls = 0;
-        $mockTest1 = AspectMock::double(TestObject::class, ['getTestActionCount' => function () use (&$numberOfCalls) {
-            $actionCount = [200, 275];
-            $result = $actionCount[$numberOfCalls];
-            $numberOfCalls++;
-
-            return $result;
-        }])->make();
+        $mockTest1 = AspectMock::double(
+            TestObject::class,
+            ['getEstimatedDuration' => function () use (&$numberOfCalls) {
+                $actionCount = [300, 275];
+                $result = $actionCount[$numberOfCalls];
+                $numberOfCalls++;
+
+                return $result;
+            }]
+        )->make();
 
         $mockHandler = AspectMock::double(
             TestObjectHandler::class,
@@ -92,17 +95,16 @@ public function testSortWithSuites()
 
         // perform sort
         $testSorter = new ParallelGroupSorter();
-        $actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 200);
+        $actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 500);
 
         // verify the resulting groups
-        $this->assertCount(5, $actualResult);
+        $this->assertCount(4, $actualResult);
 
         $expectedResults =  [
             1 => ['test3'],
-            2 => ['test2'],
-            3 => ['mockSuite1_0'],
-            4 => ['mockSuite1_1'],
-            5 => ['test5', 'test4', 'test1']
+            2 => ['test2','test5', 'test4'],
+            3 => ['mockSuite1_0', 'test1'],
+            4 => ['mockSuite1_1']
         ];
 
         foreach ($actualResult as $groupNum => $group) {

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -35,7 +35,7 @@ protected function configure()
             ->addArgument('name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'name(s) of specific tests to generate')
             ->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
             ->addOption("force", 'f',InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration')
-            ->addOption('lines', 'l', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size', 500)
+            ->addOption('time', 'i', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size (in minutes)', 10)
             ->addOption('tests', 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration');
     }
 
@@ -45,22 +45,27 @@ protected function configure()
      * @param InputInterface $input
      * @param OutputInterface $output
      * @return void
-     * @throws \Symfony\Component\Console\Exception\LogicException
+     * @throws \Symfony\Component\Console\Exception\LogicException|TestFrameworkException
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         $tests = $input->getArgument('name');
         $config = $input->getOption('config');
         $json = $input->getOption('tests');
         $force = $input->getOption('force');
-        $lines = $input->getOption('lines');
+        $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
         $verbose = $output->isVerbose();
 
         if ($json !== null && !json_decode($json)) {
             // stop execution if we have failed to properly parse any json passed in by the user
             throw new TestFrameworkException("JSON could not be parsed: " . json_last_error_msg());
         }
 
+        if ($config === 'parallel' && $time <= 0) {
+            // stop execution if the user has given us an invalid argument for time argument during parallel generation
+            throw new TestFrameworkException("time option cannot be less than or equal to 0");
+        }
+
         $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $verbose);
 
         // create our manifest file here
@@ -69,13 +74,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         if ($config == 'parallel') {
             /** @var ParallelTestManifest $testManifest */
-            $testManifest->createTestGroups($lines);
+            $testManifest->createTestGroups($time);
         }
 
         SuiteGenerator::getInstance()->generateAllSuites($testManifest);
         $testManifest->generate();
 
-        print "Generate Tests Command Run" . PHP_EOL;
+       $output->writeln("Generate Tests Command Run");
     }
 
     /**

src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

@@ -15,6 +15,18 @@
  */
 class TestObject
 {
+    const WAIT_TIME_ATTRIBUTE = 'time';
+
+    const TEST_ACTION_WEIGHT = [
+        'waitForPageLoad' => 1500,
+        'amOnPage' => 1000,
+        'waitForLoadingMaskToDisappear' => 500,
+        'wait' => self::WAIT_TIME_ATTRIBUTE,
+        'comment' => 5,
+        'assertCount' => 5,
+        'closeAdminNotification' => 10
+    ];
+
     /**
      * Name of the test
      *
@@ -159,28 +171,57 @@ public function getHooks()
     }
 
     /**
-     * Returns the number of a test actions contained within a single test (including before/after actions).
+     * Returns the estimated duration of a single test (including before/after actions).
      *
      * @return int
      */
-    public function getTestActionCount()
+    public function getEstimatedDuration()
     {
         // a skipped action results in a single skip being appended to the beginning of the test and no execution
         if ($this->isSkipped()) {
             return 1;
         }
 
-        $hookActions = 0;
+        $hookTime = 0;
         if (array_key_exists('before', $this->hooks)) {
-            $hookActions += count($this->hooks['before']->getActions());
+            $hookTime += $this->calculateWeightedActionTimes($this->hooks['before']->getActions());
         }
 
         if (array_key_exists('after', $this->hooks)) {
-            $hookActions += count($this->hooks['after']->getActions());
+            $hookTime += $this->calculateWeightedActionTimes($this->hooks['after']->getActions());
+        }
+
+        $testTime = $this->calculateWeightedActionTimes($this->getOrderedActions());
+
+        return $hookTime + $testTime;
+    }
+
+    /**
+     * Function which takes a set of actions and estimates time for completion based on action type.
+     *
+     * @param ActionObject[] $actions
+     * @return int
+     */
+    private function calculateWeightedActionTimes($actions)
+    {
+        $actionTime = 0;
+        // search for any actions of special type
+        foreach ($actions as $action) {
+            /** @var ActionObject $action */
+            if (array_key_exists($action->getType(), self::TEST_ACTION_WEIGHT)) {
+                $weight = self::TEST_ACTION_WEIGHT[$action->getType()];
+                if ($weight === self::WAIT_TIME_ATTRIBUTE) {
+                    $weight = intval($action->getCustomActionAttributes()[$weight]) * 1000;
+                }
+
+                $actionTime += $weight;
+                continue;
+            }
+
+            $actionTime += 50;
         }
 
-        $testActions = count($this->getOrderedActions());
-        return $hookActions + $testActions;
+        return $actionTime;
     }
 
     /**

src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php

@@ -77,7 +77,7 @@ private function parseEnvFile(): array
             $envContents = $this->parseEnvFileLines($envFile);
         }
 
-        return array_diff_key($this->parseEnvFileLines($envExampleFile), $envContents);
+        return array_merge($this->parseEnvFileLines($envExampleFile), $envContents);
     }
 
     private function parseEnvFileLines(array $file): array

src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php

@@ -70,22 +70,22 @@ public function __construct($suiteConfiguration, $testPath)
      */
     public function addTest($testObject)
     {
-        $this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getTestActionCount();
+        $this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getEstimatedDuration();
     }
 
     /**
      * Function which generates test groups based on arg passed. The function builds groups using the args as an upper
      * limit.
      *
-     * @param int $lines
+     * @param int $time
      * @return void
      */
-    public function createTestGroups($lines)
+    public function createTestGroups($time)
     {
         $this->testGroups = $this->parallelGroupSorter->getTestsGroupedBySize(
             $this->getSuiteConfig(),
             $this->testNameToSize,
-            $lines
+            $time
         );
 
         $this->suiteConfiguration = $this->parallelGroupSorter->getResultingSuiteConfig();

src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php

@@ -30,22 +30,22 @@ public function __construct()
      *
      * @param array $suiteConfiguration
      * @param array $testNameToSize
-     * @param integer $lines
+     * @param integer $time
      * @return array
      * @throws TestFrameworkException
      */
-    public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lines)
+    public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $time)
     {
         // we must have the lines argument in order to create the test groups
-        if ($lines == 0) {
+        if ($time == 0) {
             throw new TestFrameworkException(
-                "Please provide the argument '--lines' to the robo command in order to".
+                "Please provide the argument '--time' to the robo command in order to".
                 " generate grouped tests manifests for a parallel execution"
             );
         }
 
         $testGroups = [];
-        $splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $lines);
+        $splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $time);
         $splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests);
         $entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize);
         arsort($entriesForGeneration);
@@ -58,7 +58,7 @@ public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lin
                 continue;
             }
 
-            $testGroup = $this->createTestGroup($lines, $testName, $testSize, $testNameToSizeForUse);
+            $testGroup = $this->createTestGroup($time, $testName, $testSize, $testNameToSizeForUse);
             $testGroups[$nodeNumber] = $testGroup;
 
             // unset the test which have been used.
@@ -88,26 +88,29 @@ public function getResultingSuiteConfig()
      * a test to be used as a starting point, the size of a starting test, an array of tests available to be added to
      * the group.
      *
-     * @param integer $lineMaximum
+     * @param integer $timeMaximum
      * @param string $testName
      * @param integer $testSize
      * @param array $testNameToSizeForUse
      * @return array
      */
-    private function createTestGroup($lineMaximum, $testName, $testSize, $testNameToSizeForUse)
+    private function createTestGroup($timeMaximum, $testName, $testSize, $testNameToSizeForUse)
     {
         $group[$testName] = $testSize;
 
-        if ($testSize < $lineMaximum) {
-            while (array_sum($group) < $lineMaximum && !empty($testNameToSizeForUse)) {
+        if ($testSize < $timeMaximum) {
+            while (array_sum($group) < $timeMaximum && !empty($testNameToSizeForUse)) {
                 $groupSize = array_sum($group);
-                $lineGoal = $lineMaximum - $groupSize;
+                $lineGoal = $timeMaximum - $groupSize;
 
                 $testNameForUse = $this->getClosestLineCount($testNameToSizeForUse, $lineGoal);
-                $testSizeForUse = $testNameToSizeForUse[$testNameForUse];
+                if ($testNameToSizeForUse[$testNameForUse] < $lineGoal) {
+                    $testSizeForUse = $testNameToSizeForUse[$testNameForUse];
+                    $group[$testNameForUse] = $testSizeForUse;
+                }
+
                 unset($testNameToSizeForUse[$testNameForUse]);
 
-                $group[$testNameForUse] = $testSizeForUse;
             }
         }
 
@@ -127,8 +130,12 @@ private function getClosestLineCount($testGroup, $desiredValue)
         $winner = key($testGroup);
         $closestThreshold = $desiredValue;
         foreach ($testGroup as $testName => $testValue) {
-            $testThreshold =  abs($desiredValue - $testValue);
-            if ($closestThreshold > $testThreshold) {
+            // find the difference between the desired value and test candidate for the group
+            $testThreshold =  $desiredValue - $testValue;
+
+            // if we see that the gap between the desired value is non-negative and lower than the current closest make
+            // the test the winner.
+            if ($closestThreshold > $testThreshold && $testThreshold > 0) {
                 $closestThreshold = $testThreshold;
                 $winner = $testName;
             }
@@ -187,7 +194,7 @@ private function getSuiteNameToTestSize($suiteConfiguration)
             foreach ($test as $testName) {
                 $suiteNameToTestSize[$suite][$testName] = TestObjectHandler::getInstance()
                     ->getObject($testName)
-                    ->getTestActionCount();
+                    ->getEstimatedDuration();
             }
         }
 
@@ -224,30 +231,30 @@ private function getSuiteToSize($suiteNamesToTests)
      *
      * @param string $suiteName
      * @param array $tests
-     * @param integer $lineLimit
+     * @param integer $maxTime
      * @return array
      */
-    private function splitTestSuite($suiteName, $tests, $lineLimit)
+    private function splitTestSuite($suiteName, $tests, $maxTime)
     {
         arsort($tests);
-        $split_suites = [];
+        $splitSuites = [];
         $availableTests = $tests;
-        $split_count = 0;
+        $splitCount = 0;
 
         foreach ($tests as $test => $size) {
             if (!array_key_exists($test, $availableTests)) {
                 continue;
             }
 
-            $group = $this->createTestGroup($lineLimit, $test, $size, $availableTests);
-            $split_suites["{$suiteName}_${split_count}"] = $group;
-            $this->addSuiteToConfig($suiteName, "{$suiteName}_${split_count}", $group);
+            $group = $this->createTestGroup($maxTime, $test, $size, $availableTests);
+            $splitSuites["{$suiteName}_${splitCount}"] = $group;
+            $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group);
 
             $availableTests = array_diff_key($availableTests, $group);
-            $split_count++;
+            $splitCount++;
         }
 
-        return $split_suites;
+        return $splitSuites;
     }
 
     /**