Mqe 1671: doc for vault integration

CHANGELOG.md

@@ -1,6 +1,42 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.5.0
+-----
+* Traceability
+    * Allure output has been enhanced to contain new test artifacts created and used per MFTF step:
+        * `makeScreenshot` will contain an attachment under its Allure step.
+        * `seeInCurrentUrl` and all other `Url` asserts now contain an attachment with the expects vs actual comparison.
+        * `createData` and all other `Data` actions now contain attachments with `Request Body` and `Response Body`.
+* Modularity
+    * Added a new `mftf run:manifest` command to run testManifest files generated by `generate:tests`.
+        * See DevDocs for details
+    * `mftf generate/run:test` commands now implicitly generates the `suite` the test exists in.
+        * If a test exists in multiple suites, it will generate it in all suite contexts.
+    * `mftf run:test <testName>` will now only run the exact test provided, regardless of what is generated.
+* Maintainability
+    * Added an `--allow-skipped` flag that allows MFTF to ignore the `<skip>` annotation. This was added to the following commands:
+        * `generate:test`
+        * `run:test`
+        * `run:group`
+        * `run:failed`
+* Customizability
+    * `<entity>` defined in data.xml can now reference other `<entity>` directly.
+        * See DevDocs for details
+    * Added vault as an alternative credential storage.
+        * See DevDocs for details
+
+### Fixes
+* Fixed an issue where `grab` action variables were not substituting correctly when used as an element parameter.
+* Framework will not throw a descriptive error when referencing a `$persisted.field$` that does not exist.
+* MFTF test materials that `extends=""` itself will no longer cause infinite recursion.
+* Fixed an issue where a test could not reference a `$data.field$` whose casing was modified by the API that it used.
+* Fixed an issue with the default `functional.suite.yml` where it was incompatible with `symfony/yaml 4.0.0`.
+* Improved test generation performance via class refactors (`~10%` faster).
+
+### GitHub Issues/Pull requests:
+* [#377](https://github.com/magento/magento2-functional-testing-framework/pull/377) -- Non-API operations fixes
+
 2.4.4
 -----
 ### Fixes

bin/mftf

@@ -29,7 +29,7 @@ try {
 try {
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.4.4');
+    $application->setVersion('2.5.0');
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.4.4",
+    "version": "2.5.0",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

docs/data.md

@@ -173,6 +173,16 @@ The following is an example of a call in test:
 
 This action inputs data from the `name` of the `_defaultCategory` entity (for example, `simpleCategory598742365`) into the field with the locator defined in the selector of the `categoryNameInput` element of the `AdminCategoryBasicFieldSection`.
 
+You can also call data from the xml definition of a `data` tag directly:
+
+```xml
+<entity name="NewAdminUser" type="user">
+    <data key="username" unique="suffix">admin</data>
+    <data key="current_password">{{AnotherUser.current_password}}</data>  <!-- Data from another entity -->
+    <data key="current_password">{{_ENV.MAGENTO_ADMIN_PASSWORD}}</data>  <!-- ENV file reference -->
+</entity>
+```
+
 ## Reference
 
 ### entities {#entities-tag}

etc/config/command.php

@@ -12,8 +12,8 @@
     $tokenModel = $magentoObjectManager->get(\Magento\Integration\Model\Oauth\Token::class);
 
     $tokenPassedIn = urldecode($_POST['token']);
-    $command = str_replace([';', '&', '|'], '', urldecode($_POST['command']));
-    $arguments = str_replace([';', '&', '|'], '', urldecode($_POST['arguments']));
+    $command = urldecode($_POST['command']);
+    $arguments = urldecode($_POST['arguments']);
 
     // Token returned will be null if the token we passed in is invalid
     $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken();

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -36,7 +36,7 @@ protected function configure()
             InputOption::VALUE_NONE,
             'force generation and running of tests regardless of Magento Instance Configuration'
         )->addOption(
-            "allowSkipped",
+            "allow-skipped",
             'a',
             InputOption::VALUE_NONE,
             'Allows MFTF to generate and run skipped tests.'

src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php

@@ -46,7 +46,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
         $remove = $input->getOption('remove');
         $verbose = $output->isVerbose();
-        $allowSkipped = $input->getOption('allowSkipped');
+        $allowSkipped = $input->getOption('allow-skipped');
 
         // Set application configuration so we can references the user options in our framework
         MftfApplicationConfig::create(

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -71,7 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
         $remove = $input->getOption('remove');
         $verbose = $output->isVerbose();
-        $allowSkipped = $input->getOption('allowSkipped');
+        $allowSkipped = $input->getOption('allow-skipped');
 
         // Set application configuration so we can references the user options in our framework
         MftfApplicationConfig::create(

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -19,6 +19,13 @@
 
 class RunTestCommand extends BaseGenerateCommand
 {
+    /**
+     * The return code. Determined by all tests that run.
+     *
+     * @var integer
+     */
+    private $returnCode = 0;
+
     /**
      * Configures the current command.
      *
@@ -49,8 +56,6 @@ protected function configure()
      * @param OutputInterface $output
      * @return integer
      * @throws \Exception
-     *
-     * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -59,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $force = $input->getOption('force');
         $remove = $input->getOption('remove');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
-        $allowSkipped = $input->getOption('allowSkipped');
+        $allowSkipped = $input->getOption('allow-skipped');
         $verbose = $output->isVerbose();
 
         if ($skipGeneration and $remove) {
@@ -87,65 +92,89 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 '--force' => $force,
                 '--remove' => $remove,
                 '--debug' => $debug,
-                '--allowSkipped' => $allowSkipped,
+                '--allow-skipped' => $allowSkipped,
                 '-v' => $verbose
             ];
             $command->run(new ArrayInput($args), $output);
         }
-        // tests with resolved suite references
-        $resolvedTests = $this->resolveSuiteReferences($testConfiguration);
 
+        $testConfigArray = json_decode($testConfiguration, true);
+
+        if (isset($testConfigArray['tests'])) {
+            $this->runTests($testConfigArray['tests'], $output);
+        }
+
+        if (isset($testConfigArray['suites'])) {
+            $this->runTestsInSuite($testConfigArray['suites'], $output);
+        }
+
+        return $this->returnCode;
+    }
+
+    /**
+     * Run tests not referenced in suites
+     *
+     * @param array           $tests
+     * @param OutputInterface $output
+     * @return void
+     * @throws TestFrameworkException
+     */
+    private function runTests(array $tests, OutputInterface $output)
+    {
         $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
-        $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR;
-        $returnCode = 0;
-        //execute only tests specified as arguments in run command
-        foreach ($resolvedTests as $test) {
-            //set directory as suite name for tests in suite, if not set to "default"
-            if (strpos($test, ':')) {
-                list($testGroup, $testName) = explode(":", $test);
-            } else {
-                list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test];
-            }
-            $testGroup = $testGroup . DIRECTORY_SEPARATOR;
-            $testName = $testName . 'Cest.php';
-            if (!realpath($testsDirectory . $testGroup . $testName)) {
+        $testsDirectory = TESTS_MODULE_PATH .
+            DIRECTORY_SEPARATOR .
+            TestGenerator::GENERATED_DIR .
+            DIRECTORY_SEPARATOR .
+            TestGenerator::DEFAULT_DIR .
+            DIRECTORY_SEPARATOR ;
+
+        foreach ($tests as $test) {
+            $testName = $test . 'Cest.php';
+            if (!realpath($testsDirectory . $testName)) {
                 throw new TestFrameworkException(
-                    $testName . " is not available under " . $testsDirectory . $testGroup
+                    $testName . " is not available under " . $testsDirectory
                 );
             }
-            $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps';
-            $process = new Process($fullCommand);
-            $process->setWorkingDirectory(TESTS_BP);
-            $process->setIdleTimeout(600);
-            $process->setTimeout(0);
-
-            $returnCode = max($returnCode, $process->run(
-                function ($type, $buffer) use ($output) {
-                    $output->write($buffer);
-                }
-            ));
+            $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
+            $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
         }
-        return $returnCode;
     }
 
     /**
-     * Get an array of tests with resolved suite references from $testConfiguration
-     * eg: if test is referenced in a suite, it'll be stored in format suite:test
-     * @param string $testConfigurationJson
-     * @return array
+     * Run tests referenced in suites within suites' context.
+     *
+     * @param array           $suitesConfig
+     * @param OutputInterface $output
+     * @return void
      */
-    private function resolveSuiteReferences($testConfigurationJson)
+    private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
     {
-        $testConfiguration = json_decode($testConfigurationJson, true);
-        $testsArray = $testConfiguration['tests'] ?? [];
-        $suitesArray = $testConfiguration['suites'] ?? [];
-        $testArrayBuilder = [];
-
-        foreach ($suitesArray as $suite => $tests) {
-            foreach ($tests as $test) {
-                $testArrayBuilder[] = "$suite:$test";
-            }
+        $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps ';
+        //for tests in suites, run them as a group to run before and after block
+        foreach (array_keys($suitesConfig) as $suite) {
+            $fullCommand = $codeceptionCommand . " -g {$suite}";
+            $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
         }
-        return array_merge($testArrayBuilder, $testsArray);
+    }
+
+    /**
+     * Runs the codeception test command and returns exit code
+     *
+     * @param string          $command
+     * @param OutputInterface $output
+     * @return integer
+     *
+     * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     */
+    private function executeTestCommand(string $command, OutputInterface $output)
+    {
+        $process = new Process($command);
+        $process->setWorkingDirectory(TESTS_BP);
+        $process->setIdleTimeout(600);
+        $process->setTimeout(0);
+        return $process->run(function ($type, $buffer) use ($output) {
+            $output->write($buffer);
+        });
     }
 }

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -70,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $force = $input->getOption('force');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
-        $allowSkipped = $input->getOption('allowSkipped');
+        $allowSkipped = $input->getOption('allow-skipped');
         $verbose = $output->isVerbose();
 
         // Create Mftf Configuration
@@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             '--force' => $force,
             '--remove' => true,
             '--debug' => $debug,
-            '--allowSkipped' => $allowSkipped,
+            '--allow-skipped' => $allowSkipped,
             '-v' => $verbose
         ];
         $command->run(new ArrayInput($args), $output);

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -60,7 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $groups = $input->getArgument('groups');
         $remove = $input->getOption('remove');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
-        $allowSkipped = $input->getOption('allowSkipped');
+        $allowSkipped = $input->getOption('allow-skipped');
         $verbose = $output->isVerbose();
 
         if ($skipGeneration and $remove) {
@@ -87,7 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 '--force' => $force,
                 '--remove' => $remove,
                 '--debug' => $debug,
-                '--allowSkipped' => $allowSkipped,
+                '--allow-skipped' => $allowSkipped,
                 '-v' => $verbose
             ];
 

src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php

@@ -167,8 +167,6 @@ public function executeRequest($dependentEntities)
         $response = $executor->read($successRegex, $returnRegex, $returnIndex);
         $executor->close();
 
-        AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint');
-        AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers');
         AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body');
         AllureHelper::addAttachmentToLastStep(
             json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES),