diff --git a/bin/static-checks b/bin/static-checks index f1f58016f..c711726e4 100755 --- a/bin/static-checks +++ b/bin/static-checks @@ -8,7 +8,7 @@ set -e echo "===============================" echo " CODE SNIFFER" echo "===============================" -vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento +vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento --ignore=src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php vendor/bin/phpcs ./dev/tests/unit --standard=./dev/tests/static/Magento vendor/bin/phpcs ./dev/tests/verification --standard=./dev/tests/static/Magento --ignore=dev/tests/verification/_generated echo "" @@ -22,7 +22,7 @@ echo "" echo "===============================" echo " MESS DETECTOR" echo "===============================" -vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated +vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated,src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php echo "" echo "===============================" diff --git a/bin/static-checks.bat b/bin/static-checks.bat index 70cd44ceb..0b9402094 100644 --- a/bin/static-checks.bat +++ b/bin/static-checks.bat @@ -5,7 +5,7 @@ @echo off @echo ===============================PHP CODE SNIFFER REPORT=============================== -call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento +call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento --ignore=src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php call vendor\bin\phpcs .\dev\tests\unit --standard=.\dev\tests\static\Magento call vendor\bin\phpcs .\dev\tests\verification --standard=.\dev\tests\static\Magento --ignore=dev\tests\verification\_generated @@ -13,7 +13,7 @@ call vendor\bin\phpcs .\dev\tests\verification --standard=.\dev\tests\static\Mag call vendor\bin\phpcpd .\src @echo "===============================PHP MESS DETECTOR REPORT=============================== -vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated +vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated,src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php @echo ===============================MAGENTO COPYRIGHT REPORT=============================== echo msgbox "INFO:Copyright check currently not run as part of .bat implementation" > "%temp%\popup.vbs" diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php index 444749ed2..1acd31522 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php @@ -76,15 +76,6 @@ public function testSortWithSuites() AspectMock::double(TestObjectHandler::class, ['getInstance' => $mockHandler])->make(); - // mock a suite object - $mockSuite = AspectMock::double(SuiteObject::class, [ - 'getBeforeHook' => null, - 'getAfterHook' => null, - 'getName' => 'mockSuite1' - ])->make(); - $mockSuiteHandler = AspectMock::double(SuiteObjectHandler::class, ['getObject' => $mockSuite])->make(); - AspectMock::double(SuiteObjectHandler::class, ['getInstance' => $mockSuiteHandler]); - // create test to size array $sampleTestArray = [ 'test1' => 100, @@ -96,8 +87,7 @@ public function testSortWithSuites() // create mock suite references $sampleSuiteArray = [ - 'mockTest1' => ['mockSuite1'], - 'mockTest2' => ['mockSuite1'] + 'mockSuite1' => ['mockTest1', 'mockTest2'] ]; // perform sort diff --git a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php index 3e4e0a1b5..8930de98c 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php @@ -78,6 +78,26 @@ public function getAllObjects() return $this->suiteObjects; } + /** + * Function which return all tests referenced by suites. + * + * @return array + */ + public function getAllTestReferences() + { + $testsReferencedInSuites = []; + $suites = $this->getAllObjects(); + + foreach ($suites as $suite) { + /** @var SuiteObject $suite */ + $test_keys = array_keys($suite->getTests()); + $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]); + $testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName); + } + + return $testsReferencedInSuites; + } + /** * Method to parse all suite data xml into objects. * diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index f87ab5c4c..9c96742b4 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -6,14 +6,13 @@ namespace Magento\FunctionalTestingFramework\Suite; -use Magento\Framework\Phrase; -use Magento\Framework\Validator\Exception; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator; use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; +use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest; -use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Yaml\Yaml; @@ -75,15 +74,23 @@ public static function getInstance() */ public function generateAllSuites($testManifest) { - $suites = SuiteObjectHandler::getInstance()->getAllObjects(); - if (get_class($testManifest) == ParallelTestManifest::class) { - /** @var ParallelTestManifest $testManifest */ - $suites = $testManifest->getSorter()->getResultingSuites(); + $suites = array_keys(SuiteObjectHandler::getInstance()->getAllObjects()); + if ($testManifest != null) { + $suites = $testManifest->getSuiteConfig(); } - foreach ($suites as $suite) { - // during a parallel config run we must generate only after we have data around how a suite will be split - $this->generateSuiteFromObject($suite); + foreach ($suites as $suiteName => $suiteContent) { + $firstElement = array_values($suiteContent)[0]; + + // if the first element is a string we know that we simply have an array of tests + if (is_string($firstElement)) { + $this->generateSuiteFromTest($suiteName, $suiteContent); + } + + // if our first element is an array we know that we have split the suites + if (is_array($firstElement)) { + $this->generateSplitSuiteFromTest($suiteName, $suiteContent); + } } } @@ -96,11 +103,22 @@ public function getTestsReferencedInSuites() { $testsReferencedInSuites = []; $suites = SuiteObjectHandler::getInstance()->getAllObjects(); + + // see if we have a specific suite configuration. + if (!empty($this->suiteReferences)) { + $suites = array_intersect_key($suites, $this->suiteReferences); + } + foreach ($suites as $suite) { /** @var SuiteObject $suite */ $test_keys = array_keys($suite->getTests()); - $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]); + // see if we need to filter which tests we'll be generating. + if (array_key_exists($suite->getName(), $this->suiteReferences)) { + $test_keys = $this->suiteReferences[$suite->getName()] ?? $test_keys; + } + + $testToSuiteName = array_fill_keys($test_keys, [$suite->getName()]); $testsReferencedInSuites = array_merge_recursive($testsReferencedInSuites, $testToSuiteName); } @@ -117,35 +135,125 @@ public function getTestsReferencedInSuites() public function generateSuite($suiteName) { /**@var SuiteObject $suite **/ - $suite = SuiteObjectHandler::getInstance()->getObject($suiteName); - $this->generateSuiteFromObject($suite); + $this->generateSuiteFromTest($suiteName, []); } /** - * Function which takes a suite object and generates all relevant supporting files and classes. + * Function which takes a suite name and a set of test names. The function then generates all relevant supporting + * files and classes for the suite. The function takes an optional argument for suites which are split by a parallel + * run so that any pre/post conditions can be duplicated. * - * @param SuiteObject $suiteObject + * @param string $suiteName + * @param array $tests + * @param string $originalSuiteName * @return void */ - public function generateSuiteFromObject($suiteObject) + private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null) { - $suiteName = $suiteObject->getName(); $relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName; - $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath; - $groupNamespace = null; + $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath . DIRECTORY_SEPARATOR; DirSetupUtil::createGroupDir($fullPath); - $this->generateRelevantGroupTests($suiteName, $suiteObject->getTests()); - if ($suiteObject->requiresGroupFile()) { - // if the suite requires a group file, generate it and set the namespace - $groupNamespace = $this->groupClassGenerator->generateGroupClass($suiteObject); + $relevantTests = []; + if (!empty($tests)) { + $this->validateTestsReferencedInSuite($suiteName, $tests, $originalSuiteName); + foreach ($tests as $testName) { + $relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName); + } + } else { + $relevantTests = SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests(); } + $this->generateRelevantGroupTests($suiteName, $relevantTests); + $groupNamespace = $this->generateGroupFile($suiteName, $relevantTests, $originalSuiteName); + $this->appendEntriesToConfig($suiteName, $fullPath, $groupNamespace); print "Suite ${suiteName} generated to ${relativePath}.\n"; } + /** + * Function which validates tests passed in as custom configuration against the configuration defined by the user to + * prevent possible invalid test configurations from executing. + * + * @param string $suiteName + * @param array $testsReferenced + * @param string $originalSuiteName + * @return void + * @throws TestReferenceException + */ + private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $originalSuiteName) + { + $suiteRef = $originalSuiteName ?? $suiteName; + $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests(); + $invalidTestRef = null; + $errorMsg = "Cannot reference tests not declared as part of {$suiteRef}:\n "; + + array_walk($testsReferenced, function ($value) use (&$invalidTestRef, $possibleTestRef, &$errorMsg) { + if (!array_key_exists($value, $possibleTestRef)) { + $invalidTestRef.= "\t{$value}\n"; + } + }); + + if ($invalidTestRef != null) { + throw new TestReferenceException($errorMsg . $invalidTestRef); + } + } + + /** + * Function for generating split groups of tests (following a parallel execution). Takes a paralle suite config + * and generates applicable suites. + * + * @param string $suiteName + * @param array $suiteContent + * @return void + */ + private function generateSplitSuiteFromTest($suiteName, $suiteContent) + { + foreach ($suiteContent as $suiteSplitName => $tests) { + $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName); + } + } + + /** + * Function which takes a suite name, array of tests, and an original suite name. The function takes these args + * and generates a group file which captures suite level preconditions. + * + * @param string $suiteName + * @param array $tests + * @param string $originalSuiteName + * @return null|string + */ + private function generateGroupFile($suiteName, $tests, $originalSuiteName) + { + // if there's an original suite name we know that this test came from a split group. + if ($originalSuiteName) { + // create the new suite object + /** @var SuiteObject $originalSuite */ + $originalSuite = SuiteObjectHandler::getInstance()->getObject($originalSuiteName); + $suiteObject = new SuiteObject( + $suiteName, + $tests, + [], + $originalSuite->getHooks() + ); + } else { + $suiteObject = SuiteObjectHandler::getInstance()->getObject($suiteName); + // we have to handle the case when there is a custom configuration for an existing suite. + if (count($suiteObject->getTests()) != count($tests)) { + return $this->generateGroupFile($suiteName, $tests, $suiteName); + } + } + + if (!$suiteObject->requiresGroupFile()) { + // if we do not require a group file we don't need a namespace + return null; + } + + // if the suite requires a group file, generate it and set the namespace + return $this->groupClassGenerator->generateGroupClass($suiteObject); + } + /** * Function which accepts a suite name and suite path and appends a new group entry to the codeception.yml.dist * file in order to register the set of tests as a new group. Also appends group object location if required @@ -219,7 +327,7 @@ private static function clearPreviousSessionConfigEntries() private function generateRelevantGroupTests($path, $tests) { $testGenerator = TestGenerator::getInstance($path, $tests); - $testGenerator->createAllTestFiles('suite'); + $testGenerator->createAllTestFiles(null, []); } /** diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php index d8c7c397b..3c3eb6de9 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/BaseTestManifest.php @@ -6,6 +6,8 @@ namespace Magento\FunctionalTestingFramework\Util\Manifest; +use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; +use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; use Magento\FunctionalTestingFramework\Test\Objects\TestObject; abstract class BaseTestManifest @@ -24,16 +26,25 @@ abstract class BaseTestManifest */ protected $relativeDirPath; + /** + * Suite configuration in the format suite name to test name. Overwritten during a custom configuration. + * + * @var array + */ + protected $suiteConfiguration; + /** * TestManifest constructor. * * @param string $path * @param string $runConfig + * @param array $suiteConfiguration */ - public function __construct($path, $runConfig) + public function __construct($path, $runConfig, $suiteConfiguration) { $this->runTypeConfig = $runConfig; $this->relativeDirPath = substr($path, strlen(dirname(dirname(TESTS_BP))) + 1); + $this->suiteConfiguration = $suiteConfiguration; } /** @@ -57,9 +68,41 @@ abstract public function addTest($testObject); /** * Function which generates the actual manifest(s) once the relevant tests have been added to the array. * - * @param array $testsReferencedInSuites - * @param int|null $nodes * @return void */ - abstract public function generate($testsReferencedInSuites, $nodes = null); + abstract public function generate(); + + /** + * Getter for the suite configuration. + * + * @return array + */ + public function getSuiteConfig() + { + if ($this->suiteConfiguration === null) { + return []; + } + + $suiteToTestNames = []; + if (empty($this->suiteConfiguration)) { + // if there is no configuration passed we can assume the user wants all suites generated as specified. + foreach (SuiteObjectHandler::getInstance()->getAllObjects() as $suite => $suiteObj) { + /** @var SuiteObject $suitObj */ + $suiteToTestNames[$suite] = array_keys($suiteObj->getTests()); + } + } else { + // we need to loop through the configuration to make sure we capture suites with no specific config + foreach ($this->suiteConfiguration as $suiteName => $test) { + if (empty($test)) { + $suiteToTestNames[$suiteName] = + array_keys(SuiteObjectHandler::getInstance()->getObject($suiteName)->getTests()); + continue; + } + + $suiteToTestNames[$suiteName] = $test; + } + } + + return $suiteToTestNames; + } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php index d577be92f..f6af0fe88 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php @@ -36,16 +36,15 @@ class DefaultTestManifest extends BaseTestManifest /** * DefaultTestManifest constructor. - * @param string $manifestPath + * + * @param array $suiteConfiguration * @param string $testPath */ - public function __construct($manifestPath, $testPath) + public function __construct($suiteConfiguration, $testPath) { - $this->manifestPath = $manifestPath . DIRECTORY_SEPARATOR . 'testManifest.txt'; + $this->manifestPath = dirname($testPath) . DIRECTORY_SEPARATOR . 'testManifest.txt'; $this->cleanManifest($this->manifestPath); - parent::__construct($testPath, self::DEFAULT_CONFIG); - $fileResource = fopen($this->manifestPath, 'a'); - fclose($fileResource); + parent::__construct($testPath, self::DEFAULT_CONFIG, $suiteConfiguration); } /** @@ -62,11 +61,9 @@ public function addTest($testObject) /** * Function which outputs a list of all test files to the defined testManifest.txt file. * - * @param array $testsReferencedInSuites - * @param int|null $nodes * @return void */ - public function generate($testsReferencedInSuites, $nodes = null) + public function generate() { $fileResource = fopen($this->manifestPath, 'a'); @@ -75,7 +72,7 @@ public function generate($testsReferencedInSuites, $nodes = null) fwrite($fileResource, $line . PHP_EOL); } - $this->generateSuiteEntries($testsReferencedInSuites, $fileResource); + $this->generateSuiteEntries($fileResource); fclose($fileResource); } @@ -83,19 +80,12 @@ public function generate($testsReferencedInSuites, $nodes = null) /** * Function which takes the test suites passed to the manifest and generates corresponding entries in the manifest. * - * @param array $testsReferencedInSuites * @param resource $fileResource * @return void */ - protected function generateSuiteEntries($testsReferencedInSuites, $fileResource) + protected function generateSuiteEntries($fileResource) { - // get the names of available suites - $suiteNames = []; - array_walk($testsReferencedInSuites, function ($value) use (&$suiteNames) { - $suiteNames = array_unique(array_merge($value, $suiteNames)); - }); - - foreach ($suiteNames as $suiteName) { + foreach ($this->getSuiteConfig() as $suiteName => $tests) { $line = "-g {$suiteName}"; fwrite($fileResource, $line . PHP_EOL); } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php index 9c26e270c..12ce5d701 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Util\Manifest; +use Codeception\Suite; use Magento\Framework\Exception\RuntimeException; use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; @@ -13,6 +14,8 @@ use Magento\FunctionalTestingFramework\Test\Objects\TestObject; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\Sorter\ParallelGroupSorter; +use RecursiveArrayIterator; +use RecursiveIteratorIterator; class ParallelTestManifest extends BaseTestManifest { @@ -25,6 +28,13 @@ class ParallelTestManifest extends BaseTestManifest */ private $testNameToSize = []; + /** + * Class variable to store resulting group config. + * + * @var array + */ + private $testGroups; + /** * An instance of the group sorter which will take suites and tests organizing them to be run together. * @@ -42,14 +52,14 @@ class ParallelTestManifest extends BaseTestManifest /** * TestManifest constructor. * - * @param string $manifestPath + * @param array $suiteConfiguration * @param string $testPath */ - public function __construct($manifestPath, $testPath) + public function __construct($suiteConfiguration, $testPath) { - $this->dirPath = $manifestPath . DIRECTORY_SEPARATOR . 'groups'; + $this->dirPath = dirname($testPath) . DIRECTORY_SEPARATOR . 'groups'; $this->parallelGroupSorter = new ParallelGroupSorter(); - parent::__construct($testPath, self::PARALLEL_CONFIG); + parent::__construct($testPath, self::PARALLEL_CONFIG, $suiteConfiguration); } /** @@ -64,23 +74,35 @@ public function addTest($testObject) } /** - * Function which generates the actual manifest once the relevant tests have been added to the array. + * Function which generates test groups based on arg passed. The function builds groups using the args as an upper + * limit. * - * @param array $testsReferencedInSuites * @param int $lines * @return void */ - public function generate($testsReferencedInSuites, $lines = null) + public function createTestGroups($lines) { - DirSetupUtil::createGroupDir($this->dirPath); - $testGroups = $this->parallelGroupSorter->getTestsGroupedBySize( - $testsReferencedInSuites, + $this->testGroups = $this->parallelGroupSorter->getTestsGroupedBySize( + $this->getSuiteConfig(), $this->testNameToSize, $lines ); - foreach ($testGroups as $groupNumber => $groupContents) { - $this->generateGroupFile($groupContents, $groupNumber); + $this->suiteConfiguration = $this->parallelGroupSorter->getResultingSuiteConfig(); + } + + /** + * Function which generates the actual manifest once the relevant tests have been added to the array. + * + * @return void + */ + public function generate() + { + DirSetupUtil::createGroupDir($this->dirPath); + $suites = $this->getFlattenedSuiteConfiguration($this->suiteConfiguration ?? []); + + foreach ($this->testGroups as $groupNumber => $groupContents) { + $this->generateGroupFile($groupContents, $groupNumber, $suites); } } @@ -96,15 +118,16 @@ public function getSorter() /** * Function which takes an array containing entries representing the test execution as well as the associated group - * for the entry in order to generate a txt file used by devops for parllel execution in Jenkins. + * for the entry in order to generate a txt file used by devops for parllel execution in Jenkins. The results + * are checked against a flattened list of suites in order to generate proper entries. * * @param array $testGroup * @param int $nodeNumber + * @param array $suites * @return void */ - private function generateGroupFile($testGroup, $nodeNumber) + private function generateGroupFile($testGroup, $nodeNumber, $suites) { - $suites = $this->parallelGroupSorter->getResultingSuites(); foreach ($testGroup as $entryName => $testValue) { $fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeNumber}.txt", 'a'); @@ -118,4 +141,27 @@ private function generateGroupFile($testGroup, $nodeNumber) fclose($fileResource); } } + + /** + * Function which recusrively parses a given potentially multidimensional array of suites containing their split + * groups. The result is a flattened array of suite names to relevant tests for generation of the manifest. + * + * @param array $multiDimensionalSuites + * @return array + */ + private function getFlattenedSuiteConfiguration($multiDimensionalSuites) + { + $suites = []; + foreach ($multiDimensionalSuites as $suiteName => $suiteContent) { + $value = array_values($suiteContent)[0]; + if (is_array($value)) { + $suites = array_merge($suites, $this->getFlattenedSuiteConfiguration($suiteContent)); + continue; + } + + $suites[$suiteName] = $suiteContent; + } + + return $suites; + } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php index f4fb33b5e..94004e3be 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php @@ -12,30 +12,27 @@ class SingleRunTestManifest extends DefaultTestManifest /** * SingleRunTestManifest constructor. - * @param string $manifestPath + * + * @param array $suiteConfiguration * @param string $testPath */ - public function __construct($manifestPath, $testPath) + public function __construct($suiteConfiguration, $testPath) { - parent::__construct($manifestPath, $testPath); + parent::__construct($suiteConfiguration, $testPath); $this->runTypeConfig = self::SINGLE_RUN_CONFIG; - $fileResource = fopen($this->manifestPath, 'a'); - fclose($fileResource); } /** * Function which generates the actual manifest once the relevant tests have been added to the array. * - * @param array $testsReferencedInSuites - * @param int|null $nodes * @return void */ - public function generate($testsReferencedInSuites, $nodes = null) + public function generate() { $fileResource = fopen($this->manifestPath, 'a'); $line = $this->relativeDirPath . DIRECTORY_SEPARATOR; fwrite($fileResource, $line . PHP_EOL); - $this->generateSuiteEntries($testsReferencedInSuites, $fileResource); + $this->generateSuiteEntries($fileResource); fclose($fileResource); } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php index 4344a96a1..51b34bc80 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php @@ -6,6 +6,9 @@ namespace Magento\FunctionalTestingFramework\Util\Manifest; +use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; +use Magento\FunctionalTestingFramework\Util\TestGenerator; + class TestManifestFactory { /** @@ -19,26 +22,28 @@ private function __construct() /** * Static function which takes path and config to return the appropriate manifest output type. * - * @param String $manifestPath - * @param String $testPath * @param String $runConfig + * @param array $suiteConfiguration + * @param String $testPath * @return BaseTestManifest */ - public static function makeManifest($manifestPath, $testPath, $runConfig) + public static function makeManifest($runConfig, $suiteConfiguration, $testPath = TestGenerator::DEFAULT_DIR) { + $testDirFullPath = TESTS_MODULE_PATH + . DIRECTORY_SEPARATOR + . TestGenerator::GENERATED_DIR + . DIRECTORY_SEPARATOR + . $testPath; + switch ($runConfig) { case 'singleRun': - return new SingleRunTestManifest($manifestPath, $testPath); + return new SingleRunTestManifest($suiteConfiguration, $testDirFullPath); case 'parallel': - return new ParallelTestManifest($manifestPath, $testPath); - - case 'suite': - // the suite does not have its own manifest but instead is handled by the other suite types. - return null; + return new ParallelTestManifest($suiteConfiguration, $testDirFullPath); default: - return new DefaultTestManifest($manifestPath, $testPath); + return new DefaultTestManifest($suiteConfiguration, $testDirFullPath); } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php index 9d51bf2e1..dfe5d3915 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php +++ b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php @@ -6,8 +6,6 @@ namespace Magento\FunctionalTestingFramework\Util\Sorter; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; -use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; -use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; class ParallelGroupSorter @@ -17,7 +15,7 @@ class ParallelGroupSorter * * @var array */ - private $compositeSuiteObjects = []; + private $suiteConfig = []; /** * ParallelGroupSorter constructor. @@ -30,13 +28,13 @@ public function __construct() /** * Function which returns tests and suites split according to desired number of lines divded into groups. * - * @param array $testNameToSuiteName + * @param array $suiteConfiguration * @param array $testNameToSize * @param integer $lines * @return array * @throws TestFrameworkException */ - public function getTestsGroupedBySize($testNameToSuiteName, $testNameToSize, $lines) + public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lines) { // we must have the lines argument in order to create the test groups if ($lines == 0) { @@ -47,7 +45,7 @@ public function getTestsGroupedBySize($testNameToSuiteName, $testNameToSize, $li } $testGroups = []; - $splitSuiteNamesToTests = $this->createGroupsWithinSuites($testNameToSuiteName, $lines); + $splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $lines); $splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests); $entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize); arsort($entriesForGeneration); @@ -64,7 +62,7 @@ public function getTestsGroupedBySize($testNameToSuiteName, $testNameToSize, $li $testGroups[$nodeNumber] = $testGroup; // unset the test which have been used. - $testNameToSizeForUse = array_diff($testNameToSizeForUse, $testGroup); + $testNameToSizeForUse = array_diff_key($testNameToSizeForUse, $testGroup); $nodeNumber++; } @@ -76,9 +74,13 @@ public function getTestsGroupedBySize($testNameToSuiteName, $testNameToSize, $li * * @return array */ - public function getResultingSuites() + public function getResultingSuiteConfig() { - return $this->compositeSuiteObjects; + if (empty($this->suiteConfig)) { + return null; + } + + return $this->suiteConfig; } /** @@ -139,22 +141,14 @@ private function getClosestLineCount($testGroup, $desiredValue) * Function which takes an array of test names mapped to suite name and a size limitation for each group of tests. * The function divides suites that are over the specified limit and returns the resulting suites in an array. * - * @param array $testNameToSuiteName + * @param array $suiteConfiguration * @param integer $lineLimit * @return array */ - private function createGroupsWithinSuites($testNameToSuiteName, $lineLimit) + private function createGroupsWithinSuites($suiteConfiguration, $lineLimit) { - $suiteNameToTestNames = []; - $suiteNameToSize = []; - array_walk($testNameToSuiteName, function ($value, $key) use (&$suiteNameToTestNames, &$suiteNameToSize) { - $testActionCount = TestObjectHandler::getInstance()->getObject($key)->getTestActionCount(); - foreach ($value as $suite) { - $suiteNameToTestNames[$suite][$key] = $testActionCount; - $currentSize = $suiteNameToSize[$suite] ?? 0; - $suiteNameToSize[$suite] = $currentSize + $testActionCount; - } - }); + $suiteNameToTestSize = $this->getSuiteNameToTestSize($suiteConfiguration); + $suiteNameToSize = $this->getSuiteToSize($suiteNameToTestSize); // divide the suites up within the array $suitesForResize = array_filter($suiteNameToSize, function ($val) use ($lineLimit) { @@ -162,17 +156,17 @@ private function createGroupsWithinSuites($testNameToSuiteName, $lineLimit) }); // remove the suites for resize from the original list - $remainingSuites = array_diff_key($suiteNameToTestNames, $suitesForResize); + $remainingSuites = array_diff_key($suiteNameToTestSize, $suitesForResize); foreach ($remainingSuites as $remainingSuite => $tests) { - $this->addSuiteAsObject($remainingSuite, null, null); + $this->addSuiteToConfig($remainingSuite, null, $tests); } $resultingGroups = []; foreach ($suitesForResize as $suiteName => $suiteSize) { $resultingGroups = array_merge( $resultingGroups, - $this->splitTestSuite($suiteName, $suiteNameToTestNames[$suiteName], $lineLimit) + $this->splitTestSuite($suiteName, $suiteNameToTestSize[$suiteName], $lineLimit) ); } @@ -180,6 +174,26 @@ private function createGroupsWithinSuites($testNameToSuiteName, $lineLimit) return array_merge($remainingSuites, $resultingGroups); } + /** + * Function which takes the given suite configuration and returns an array of suite to test size. + * + * @param array $suiteConfiguration + * @return array + */ + private function getSuiteNameToTestSize($suiteConfiguration) + { + $suiteNameToTestSize = []; + foreach ($suiteConfiguration as $suite => $test) { + foreach ($test as $testName) { + $suiteNameToTestSize[$suite][$testName] = TestObjectHandler::getInstance() + ->getObject($testName) + ->getTestActionCount(); + } + } + + return $suiteNameToTestSize; + } + /** * Function which takes a multidimensional array containing a suite name mapped to an array of tests names as keys * with their sizes as values. The function returns an array of suite name to size of the corresponding mapped @@ -227,7 +241,7 @@ private function splitTestSuite($suiteName, $tests, $lineLimit) $group = $this->createTestGroup($lineLimit, $test, $size, $availableTests); $split_suites["{$suiteName}_${split_count}"] = $group; - $this->addSuiteAsObject($suiteName, "{$suiteName}_${split_count}", $group); + $this->addSuiteToConfig($suiteName, "{$suiteName}_${split_count}", $group); $availableTests = array_diff($availableTests, $group); $split_count++; @@ -246,25 +260,13 @@ private function splitTestSuite($suiteName, $tests, $lineLimit) * @param array $tests * @return void */ - private function addSuiteAsObject($originalSuiteName, $newSuiteName, $tests) + private function addSuiteToConfig($originalSuiteName, $newSuiteName, $tests) { - /** @var SuiteObject $originalSuite */ - $originalSuite = SuiteObjectHandler::getInstance()->getObject($originalSuiteName); - if ($newSuiteName == null && $tests == null) { - $this->compositeSuiteObjects[$originalSuiteName] = $originalSuite; + if ($newSuiteName == null) { + $this->suiteConfig[$originalSuiteName] = array_keys($tests); return; } - $newSuiteTests = []; - foreach ($tests as $test => $lines) { - $newSuiteTests[$test] = TestObjectHandler::getInstance()->getObject($test); - } - - $this->compositeSuiteObjects[$newSuiteName] = new SuiteObject( - $newSuiteName, - $newSuiteTests, - [], - $originalSuite->getHooks() - ); + $this->suiteConfig[$originalSuiteName][$newSuiteName] = array_keys($tests); } } diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index e1b15791f..dfbf20dac 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -8,6 +8,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject; @@ -96,7 +97,7 @@ private function __construct($exportDir, $tests, $debug = false) * @param bool $debug * @return TestGenerator */ - public static function getInstance($dir = null, $tests = null, $debug = false) + public static function getInstance($dir = null, $tests = [], $debug = false) { return new TestGenerator($dir, $tests, $debug); } @@ -112,14 +113,28 @@ public function getExportDir() } /** - * Load all Test files as Objects using the Test Object Handler. + * Load all Test files as Objects using the Test Object Handler, additionally validates test references being loaded + * for validity. * + * @param array $testsToIgnore * @return array */ - private function loadAllTestObjects() + private function loadAllTestObjects($testsToIgnore) { if ($this->tests === null || empty($this->tests)) { - return TestObjectHandler::getInstance()->getAllObjects(); + $testObjects = TestObjectHandler::getInstance()->getAllObjects(); + return array_diff_key($testObjects, $testsToIgnore); + } + + // If we have a custom configuration, we need to check the tests passed in to insure that we can generate + // them in the current context. + $invalidTestObjects = array_intersect_key($this->tests, $testsToIgnore); + if (!empty($invalidTestObjects)) { + $errorMsg = "Cannot reference the following tests for generation without accompanying suite:\n"; + array_walk($invalidTestObjects, function ($value, $key) use (&$errorMsg) { + $errorMsg.= "\t{$key}\n"; + }); + throw new TestReferenceException($errorMsg); } return $this->tests; @@ -151,30 +166,28 @@ private function createCestFile($testPhp, $filename) * Assemble ALL PHP strings using the assembleAllTestPhp function. Loop over and pass each array item * to the createCestFile function. * - * @param string $runConfig - * @param int $nodes - * @param TestObject[] $testsToIgnore - * @return BaseTestManifest + * @param BaseTestManifest $testManifest + * @param array $testsToIgnore + * @return void * @throws TestReferenceException * @throws \Exception */ - public function createAllTestFiles($runConfig = null, $nodes = null, $testsToIgnore = []) + public function createAllTestFiles($testManifest = null, $testsToIgnore = null) { - DirSetupUtil::createGroupDir($this->exportDirectory); + if ($this->tests === null) { + // no-op if the test configuration is null + return; + } - // create our manifest file here - $testManifest = TestManifestFactory::makeManifest( - dirname($this->exportDirectory), - $this->exportDirectory, - $runConfig - ); + DirSetupUtil::createGroupDir($this->exportDirectory); + if ($testsToIgnore === null) { + $testsToIgnore = SuiteObjectHandler::getInstance()->getAllTestReferences(); + } - $testPhpArray = $this->assembleAllTestPhp($testManifest, $nodes, $testsToIgnore); + $testPhpArray = $this->assembleAllTestPhp($testManifest, $testsToIgnore); foreach ($testPhpArray as $testPhpFile) { $this->createCestFile($testPhpFile[1], $testPhpFile[0]); } - - return $testManifest; } /** @@ -216,15 +229,13 @@ private function assembleTestPhp($testObject) * Load ALL Test objects. Loop over and pass each to the assembleTestPhp function. * * @param BaseTestManifest $testManifest - * @param int $nodes * @param array $testsToIgnore * @return array */ - private function assembleAllTestPhp($testManifest, $nodes, array $testsToIgnore) + private function assembleAllTestPhp($testManifest, array $testsToIgnore) { /** @var TestObject[] $testObjects */ - $testObjects = $this->loadAllTestObjects(); - $testObjects = array_diff_key($testObjects, $testsToIgnore); + $testObjects = $this->loadAllTestObjects($testsToIgnore); $cestPhpArray = []; foreach ($testObjects as $test) { @@ -242,10 +253,6 @@ private function assembleAllTestPhp($testManifest, $nodes, array $testsToIgnore) } } - if ($testManifest != null) { - $testManifest->generate($testsToIgnore, intval($nodes)); - } - return $cestPhpArray; }