diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f82a2f9..6471920c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,42 @@ Magento Functional Testing Framework Changelog ================================================ +2.5.0 +----- +* Traceability + * Allure output has been enhanced to contain new test artifacts created and used per MFTF step: + * `makeScreenshot` will contain an attachment under its Allure step. + * `seeInCurrentUrl` and all other `Url` asserts now contain an attachment with the expects vs actual comparison. + * `createData` and all other `Data` actions now contain attachments with `Request Body` and `Response Body`. +* Modularity + * Added a new `mftf run:manifest` command to run testManifest files generated by `generate:tests`. + * See DevDocs for details + * `mftf generate/run:test` commands now implicitly generates the `suite` the test exists in. + * If a test exists in multiple suites, it will generate it in all suite contexts. + * `mftf run:test ` will now only run the exact test provided, regardless of what is generated. +* Maintainability + * Added an `--allow-skipped` flag that allows MFTF to ignore the `` annotation. This was added to the following commands: + * `generate:test` + * `run:test` + * `run:group` + * `run:failed` +* Customizability + * `` defined in data.xml can now reference other `` directly. + * See DevDocs for details + * Added vault as an alternative credential storage. + * See DevDocs for details + +### Fixes +* Fixed an issue where `grab` action variables were not substituting correctly when used as an element parameter. +* Framework will not throw a descriptive error when referencing a `$persisted.field$` that does not exist. +* MFTF test materials that `extends=""` itself will no longer cause infinite recursion. +* Fixed an issue where a test could not reference a `$data.field$` whose casing was modified by the API that it used. +* Fixed an issue with the default `functional.suite.yml` where it was incompatible with `symfony/yaml 4.0.0`. +* Improved test generation performance via class refactors (`~10%` faster). + +### GitHub Issues/Pull requests: +* [#377](https://github.com/magento/magento2-functional-testing-framework/pull/377) -- Non-API operations fixes + 2.4.4 ----- ### Fixes diff --git a/bin/mftf b/bin/mftf index 0f2bf274d..7f9db3524 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.4.4'); + $application->setVersion('2.5.0'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 66936938c..b33567303 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.4.4", + "version": "2.5.0", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 10169fb08..dd7d6ad3f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b7f01a11ff57f3b64f08352c33a506f", + "content-hash": "01cbd9e237e76de7070a3c0de4ee8b9f", "packages": [ { "name": "allure-framework/allure-codeception", @@ -250,7 +250,7 @@ { "name": "Tobias Nyholm", "email": "tobias.nyholm@gmail.com", - "homepage": "https://github.com/nyholm" + "homepage": "https://github.com/Nyholm" } ], "description": "Library of all the php-cache adapters", diff --git a/docs/data.md b/docs/data.md index 875df0198..4c618ae45 100644 --- a/docs/data.md +++ b/docs/data.md @@ -173,6 +173,16 @@ The following is an example of a call in test: This action inputs data from the `name` of the `_defaultCategory` entity (for example, `simpleCategory598742365`) into the field with the locator defined in the selector of the `categoryNameInput` element of the `AdminCategoryBasicFieldSection`. +You can also call data from the xml definition of a `data` tag directly: + +```xml + + admin + {{AnotherUser.current_password}} + {{_ENV.MAGENTO_ADMIN_PASSWORD}} + +``` + ## Reference ### entities {#entities-tag} diff --git a/etc/config/command.php b/etc/config/command.php index f018e3014..047af324a 100644 --- a/etc/config/command.php +++ b/etc/config/command.php @@ -12,8 +12,8 @@ $tokenModel = $magentoObjectManager->get(\Magento\Integration\Model\Oauth\Token::class); $tokenPassedIn = urldecode($_POST['token']); - $command = str_replace([';', '&', '|'], '', urldecode($_POST['command'])); - $arguments = str_replace([';', '&', '|'], '', urldecode($_POST['arguments'])); + $command = urldecode($_POST['command']); + $arguments = urldecode($_POST['arguments']); // Token returned will be null if the token we passed in is invalid $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken(); diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 40582db86..87b203d66 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -36,7 +36,7 @@ protected function configure() InputOption::VALUE_NONE, 'force generation and running of tests regardless of Magento Instance Configuration' )->addOption( - "allowSkipped", + "allow-skipped", 'a', InputOption::VALUE_NONE, 'Allows MFTF to generate and run skipped tests.' diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php index 7dd3b8cb4..cd798f420 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php @@ -46,7 +46,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 64c7c01d5..621f29d03 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -71,7 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b83953dce..77b8be513 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -19,6 +19,13 @@ class RunTestCommand extends BaseGenerateCommand { + /** + * The return code. Determined by all tests that run. + * + * @var integer + */ + private $returnCode = 0; + /** * Configures the current command. * @@ -49,8 +56,6 @@ protected function configure() * @param OutputInterface $output * @return integer * @throws \Exception - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ protected function execute(InputInterface $input, OutputInterface $output): int { @@ -59,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -87,65 +92,89 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); } - // tests with resolved suite references - $resolvedTests = $this->resolveSuiteReferences($testConfiguration); + $testConfigArray = json_decode($testConfiguration, true); + + if (isset($testConfigArray['tests'])) { + $this->runTests($testConfigArray['tests'], $output); + } + + if (isset($testConfigArray['suites'])) { + $this->runTestsInSuite($testConfigArray['suites'], $output); + } + + return $this->returnCode; + } + + /** + * Run tests not referenced in suites + * + * @param array $tests + * @param OutputInterface $output + * @return void + * @throws TestFrameworkException + */ + private function runTests(array $tests, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; - $returnCode = 0; - //execute only tests specified as arguments in run command - foreach ($resolvedTests as $test) { - //set directory as suite name for tests in suite, if not set to "default" - if (strpos($test, ':')) { - list($testGroup, $testName) = explode(":", $test); - } else { - list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test]; - } - $testGroup = $testGroup . DIRECTORY_SEPARATOR; - $testName = $testName . 'Cest.php'; - if (!realpath($testsDirectory . $testGroup . $testName)) { + $testsDirectory = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + TestGenerator::GENERATED_DIR . + DIRECTORY_SEPARATOR . + TestGenerator::DEFAULT_DIR . + DIRECTORY_SEPARATOR ; + + foreach ($tests as $test) { + $testName = $test . 'Cest.php'; + if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( - $testName . " is not available under " . $testsDirectory . $testGroup + $testName . " is not available under " . $testsDirectory ); } - $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - - $returnCode = max($returnCode, $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - )); + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } - return $returnCode; } /** - * Get an array of tests with resolved suite references from $testConfiguration - * eg: if test is referenced in a suite, it'll be stored in format suite:test - * @param string $testConfigurationJson - * @return array + * Run tests referenced in suites within suites' context. + * + * @param array $suitesConfig + * @param OutputInterface $output + * @return void */ - private function resolveSuiteReferences($testConfigurationJson) + private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { - $testConfiguration = json_decode($testConfigurationJson, true); - $testsArray = $testConfiguration['tests'] ?? []; - $suitesArray = $testConfiguration['suites'] ?? []; - $testArrayBuilder = []; - - foreach ($suitesArray as $suite => $tests) { - foreach ($tests as $test) { - $testArrayBuilder[] = "$suite:$test"; - } + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; + //for tests in suites, run them as a group to run before and after block + foreach (array_keys($suitesConfig) as $suite) { + $fullCommand = $codeceptionCommand . " -g {$suite}"; + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } - return array_merge($testArrayBuilder, $testsArray); + } + + /** + * Runs the codeception test command and returns exit code + * + * @param string $command + * @param OutputInterface $output + * @return integer + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + private function executeTestCommand(string $command, OutputInterface $output) + { + $process = new Process($command); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + return $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); } } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 15ba723e6..336fd5d87 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -70,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); // Create Mftf Configuration @@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => true, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 20d290af9..7f954c8fe 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -60,7 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -87,7 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 4b184c206..691ce3606 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -167,8 +167,6 @@ public function executeRequest($dependentEntities) $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); - AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); - AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body'); AllureHelper::addAttachmentToLastStep( json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES),