Merge branch 'develop' into MQE-1534

Author: KevinBKozan

docs/commands/mftf.md

@@ -68,6 +68,7 @@ This command cleans up the previously generated tests; generates and runs the te
 For more details about `failed`, refer to [Reporting][].
 
 ### Generate documentation for action groups
+
 ```bash
 vendor/bin/mftf generate:docs
 ```
@@ -115,14 +116,14 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 
 #### Options
 
-| Option                                        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| --------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--config=[<default>|<singleRun>|<parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.</br> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.</br> Example: `generate:tests --config=parallel`. |
-| `--force`                                     | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`.                                                                                                                                                                                                                                                                                                                                                             |
-| `-i,--time`                                     | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`                                                                                                                                                                                                                                                                                                            |
-| `--tests`                                     | Defines the test configuration as a JSON string.                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| `--debug`                                     | Returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This parameter takes extra processing time. Use it after test generation has failed once.                                                                                                                                                                                                                                                   |
-| `-r,--remove`                                 | Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.                                                                                                                                                                                                                                                 |
+| Option | Description|
+| ---| --- |
+| `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.</br> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.</br> Example: `generate:tests --config=parallel`. |
+| `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
+| `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
+| `--tests` | Defines the test configuration as a JSON string.|
+| `--debug` | Returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This parameter takes extra processing time. Use it after test generation has failed once. |
+| `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.|
 
 #### Examples of the JSON configuration
 
@@ -131,7 +132,7 @@ The configuration to generate a single test with no suites:
 ```json
 {  
    "tests":[
-      "general_test1"     //Generate the "general_test1" test.
+      "general_test1"  //Generate the "general_test1" test.
       ],
    "suites": null
 }
@@ -141,10 +142,10 @@ The configuration to generate a single test in the suite:
 
 ```json
 {  
-   "tests": null,         // No tests outside the suite configuration will be generated.
+   "tests": null,       // No tests outside the suite configuration will be generated.
    "suites":{  
-      "sample":[          // The suite that contains the test.
-         "suite_test1"    // The test to be generated.
+      "sample":[        // The suite that contains the test.
+         "suite_test1"  // The test to be generated.
       ]
    }
 }
@@ -159,11 +160,11 @@ Complex configuration to generate a few non-suite tests, a single test in a suit
       "general_test2",
       "general_test3"
    ],
-   "suites":{             //Go to suites.
-      "sample":[          //Go to the "sample" suite.
-         "suite_test1"    //Generate the "suite_test1" test.
+   "suites":{           //Go to suites.
+      "sample":[        //Go to the "sample" suite.
+         "suite_test1"  //Generate the "suite_test1" test.
       ],
-      "sample2":[]        //Generate all tests in the "sample2" suite.
+      "sample2":[]      //Generate all tests in the "sample2" suite.
    }
 }
 ```
@@ -190,8 +191,8 @@ vendor/bin/mftf generate:suite <suite name> [<suite name>] [--remove]
 
 #### Options
 
-| Option        | Description                                                                                                                                                                                                                                            |
-| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Option | Description |
+| --- | --- |
 | `-r,--remove` | Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `vendor/bin/mftf generate:suite WYSIWYG --remove` cleans up the entire `_generated` directory and generates `WYSIWYG` only. |
 
 #### Example
@@ -253,7 +254,6 @@ vendor/bin/mftf generate:docs [--clean] [--output=/path/to/alternate/dir]
 vendor/bin/mftf generate:docs --clean
 ```
 
-
 ### `reset`
 
 #### Description
@@ -389,6 +389,20 @@ vendor/bin/mftf setup:env
 
 The example parameters are taken from the `etc/config/.env.example` file.
 
+### `static:checks`
+
+Runs all MFTF static:checks on the test codebase that MFTF is currently attached to.
+
+#### Existing static checks
+
+* Test Dependency: Checks that test dependencies do not violate Magento module's composer dependencies.
+
+#### Usage
+
+```bash
+vendor/bin/mftf static:checks
+```
+
 ### `upgrade:tests`
 
 Applies all the MFTF major version upgrade scripts to test components in the given path (`test.xml`, `data.xml`, etc).

docs/tips-tricks.md

@@ -283,6 +283,27 @@ Use numbers within `stepKeys` when order is important, such as with testing sort
 
 ## Selectors
 
+### Use contains() around text()
+
+When possible, use `contains(text(), 'someTextHere')` rather than `text()='someTextHere'`.
+`contains()` ignores whitespace while `text()` accounts for it.
+
+**Why?**
+If you are comparing text within a selector and have an unexpected space, or a blank line above or below the string, `text()` will fail while the `contains(text())` format will catch it.
+In this scenario `text()` is more exacting. Use it when you need to be very precise about what is getting compared.
+
+<span style="color:green">
+GOOD:
+</span>
+
+`//span[contains(text(), 'SomeTextHere')]`
+
+<span style="color:red">
+BAD:
+</span>
+
+`//span[text()='SomeTextHere']`
+
 ### Build selectors in proper order
 
 When building selectors for form elements, start with the parent context of the form element.
@@ -353,6 +374,31 @@ BAD:
 
 ## General tips
 
+### Use data references to avoid hardcoded values
+
+If you need to run a command such as  `<magentoCLI command="config:set" />`, do not hardcode paths and values to the command.
+Rather, create an appropriate `ConfigData.xml` file, which contains the required parameters for running the command.
+It will simplify the future maintanence of tests.
+
+ <span style="color:green">
+GOOD:
+</span>
+
+```xml
+<magentoCLI command="config:set {{StorefrontCustomerCaptchaLength3ConfigData.path}} {{StorefrontCustomerCaptchaLength3ConfigData.value}}" stepKey="setCaptchaLength" />
+```
+
+ <span style="color:red">
+BAD:
+</span>
+
+```xml
+<magentoCLI command="config:set customer/captcha/length 3" stepKey="setCaptchaLength" />
+```
+
+For example:
+[This test][] refers to this [Data file][].
+
 ### Use descriptive variable names
 
 Use descriptive variable names to increase readability.
@@ -398,3 +444,7 @@ BAD:
 ```
 
 <!--{% endraw %}-->
+
+<!-- Link Definitions -->
+[This test]: https://github.com/magento/magento2/blob/2.3-develop/app/code/Magento/Captcha/Test/Mftf/Test/StorefrontCaptchaRegisterNewCustomerTest.xml#L24
+[Data file]: https://github.com/magento/magento2/blob/2.3-develop/app/code/Magento/Captcha/Test/Mftf/Data/CaptchaConfigData.xml

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -46,12 +46,12 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return integer|null|void
+     * @return integer
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $tests = $input->getArgument('name');
         $skipGeneration = $input->getOption('skip-generate');
@@ -85,7 +85,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $process->setWorkingDirectory(TESTS_BP);
         $process->setIdleTimeout(600);
         $process->setTimeout(0);
-        $process->run(
+
+        return $process->run(
             function ($type, $buffer) use ($output) {
                 $output->write($buffer);
             }

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -59,13 +59,13 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return integer|null|void
+     * @return integer
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
         // Create Mftf Configuration
         MftfApplicationConfig::create(
@@ -78,7 +78,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $testConfiguration = $this->getFailedTestList();
 
         if ($testConfiguration === null) {
-            return null;
+            // no failed tests found, run is a success
+            return 0;
         }
 
         $command = $this->getApplication()->find('generate:tests');
@@ -87,7 +88,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $command->run(new ArrayInput($args), $output);
 
         $testManifestList = $this->readTestManifestFile();
-
+        $returnCode = 0;
         foreach ($testManifestList as $testCommand) {
             $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
             $codeceptionCommand .= $testCommand;
@@ -96,11 +97,11 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $process->setWorkingDirectory(TESTS_BP);
             $process->setIdleTimeout(600);
             $process->setTimeout(0);
-            $process->run(
+            $returnCode = max($returnCode, $process->run(
                 function ($type, $buffer) use ($output) {
                     $output->write($buffer);
                 }
-            );
+            ));
             if (file_exists(self::TESTS_FAILED_FILE)) {
                 $this->failedList = array_merge(
                     $this->failedList,
@@ -111,6 +112,8 @@ function ($type, $buffer) use ($output) {
         foreach ($this->failedList as $test) {
             $this->writeFailedTestToFile($test, self::TESTS_FAILED_FILE);
         }
+
+        return $returnCode;
     }
 
     /**

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -53,12 +53,12 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return integer|null|void
+     * @return integer
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $skipGeneration = $input->getOption('skip-generate');
         $force = $input->getOption('force');
@@ -102,7 +102,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $process->setWorkingDirectory(TESTS_BP);
         $process->setIdleTimeout(600);
         $process->setTimeout(0);
-        $process->run(
+
+        return $process->run(
             function ($type, $buffer) use ($output) {
                 $output->write($buffer);
             }