Merge branch 'develop' into MQE-1600

Author: jilu1

.gitignore

@@ -16,4 +16,6 @@ dev/tests/functional/MFTF.suite.yml
 dev/tests/functional/_output
 dev/mftf.log
 dev/tests/mftf.log
-dev/tests/docs/*
+dev/tests/docs/*
+dev/tests/_output
+dev/tests/functional.suite.yml

docs/selectors.md

@@ -0,0 +1,35 @@
+## Selectors
+
+These guidelines should help you to write high quality selectors.
+
+### Selectors SHOULD be written in CSS instead of XPath whenever possible
+
+CSS is generally easier to read than XPath. For example, `//*[@id="foo"]` in XPath can be expressed as simply as `#foo` in CSS.
+See this [XPath Cheatsheet](https://devhints.io/xpath) for more examples.
+
+### XPath selectors SHOULD NOT use `@attribute="foo"`. 
+
+This would fail if the attribute was `attribute="foo bar"`.
+Instead you SHOULD use `contains(@attribute, "foo")` where `@attribute` is any valid attribute such as `@text` or `@class`.
+
+### CSS and XPath selectors SHOULD be implemented in their most simple form
+
+* <span class="color:green">GOOD:</span> `#foo`
+* <span class="color:red">BAD:</span> `button[contains(@id, "foo")]`
+
+### CSS and XPath selectors SHOULD avoid making use of hardcoded indices
+
+Instead you SHOULD parameterize the selector.
+
+* <span class="color:green">GOOD:</span> `.foo:nth-of-type({{index}})`
+* <span class="color:red">BAD:</span> `.foo:nth-of-type(1)`
+
+* <span class="color:green">GOOD:</span> `button[contains(@id, "foo")][{{index}}]`
+* <span class="color:red">BAD:</span> `button[contains(@id, "foo")][1]`
+
+* <span class="color:green">GOOD:</span> `#actions__{{index}}__aggregator`
+* <span class="color:red">BAD:</span> `#actions__1__aggregator`
+
+### CSS and XPath selectors MUST NOT reference the `@data-bind` attribute
+
+The `@data-bind` attribute is used by KnockoutJS, a framework Magento uses to create dynamic Javascript pages. Since this `@data-bind` attribute is tied to a specific framework, it should not be used for selectors. If Magento decides to use a different framework then these `@data-bind` selectors would break.

docs/troubleshooting.md

@@ -41,7 +41,7 @@ Chrome v62 is in the process of being rolled out, and it causes an error with Ch
 
 #### Solution
 
-Use [ChromeDriver v2.33+][]{:target="\_blank"} and [Selenium Server Standalone v3.6.0+][]{:target="\_blank"} in order to execute tests in Google Chrome v62+.
+Use [ChromeDriver 74.0.3729.6+][]{:target="\_blank"} and [Selenium Server Standalone v3.9+][]{:target="\_blank"} in order to execute tests in Google Chrome v62+.
 
 ### Firefox
 
@@ -57,5 +57,5 @@ None yet. Solving this problem is dependent on a GeckoDriver fix.
 
 <!-- Link Definitions -->
 [Headless Chrome]: https://developers.google.com/web/updates/2017/04/headless-chrome
-[ChromeDriver v2.33+]: https://chromedriver.storage.googleapis.com/index.html?path=2.33/
-[Selenium Server Standalone v3.6.0+]: http://www.seleniumhq.org/download/
+[ChromeDriver 74.0.3729.6+]: https://chromedriver.storage.googleapis.com/index.html?path=2.33/
+[Selenium Server Standalone v3.9+]: http://www.seleniumhq.org/download/

src/Magento/FunctionalTestingFramework/Codeception/Subscriber/Console.php

@@ -31,6 +31,20 @@ class Console extends \Codeception\Subscriber\Console
      */
     private $actionGroupStepKey = null;
 
+    /**
+     * Console constructor. Parent constructor requires codeception CLI options, and does not have its own configs.
+     * Constructor is only different than parent due to the way Codeception instantiates Extensions.
+     *
+     * @param array $extensionOptions
+     * @param array $options
+     *
+     * @SuppressWarnings(PHPMD)
+     */
+    public function __construct($extensionOptions = [], $options = [])
+    {
+        parent::__construct($options);
+    }
+
     /**
      * Printing stepKey in before step action.
      *

src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php

@@ -9,18 +9,19 @@
 namespace Magento\FunctionalTestingFramework\Console;
 
 use Magento\FunctionalTestingFramework\StaticCheck\StaticChecksList;
+use Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Output\OutputInterface;
+use Exception;
 
 class StaticChecksCommand extends Command
 {
     /**
      * Pool of static check scripts to run
      *
-     * @var \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface
+     * @var StaticCheckListInterface
      */
     private $staticChecksList;
 
@@ -41,16 +42,28 @@ protected function configure()
      *
      * @param InputInterface $input
      * @param OutputInterface $output
-     * @return int|null|void
-     * @throws \Exception
+     * @return int
+     * @throws Exception
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         $staticCheckObjects = $this->staticChecksList->getStaticChecks();
+
+        $errors = [];
+
         foreach ($staticCheckObjects as $staticCheck) {
-            $staticOutput = $staticCheck->execute($input);
+            $staticCheck->execute($input);
+
+            $staticOutput = $staticCheck->getOutput();
             LoggingUtil::getInstance()->getLogger(get_class($staticCheck))->info($staticOutput);
             $output->writeln($staticOutput);
+            $errors += $staticCheck->getErrors();
+        }
+
+        if (empty($errors)) {
+            return 0;
+        } else {
+            return 1;
         }
     }
 }

src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php

@@ -16,7 +16,19 @@ interface StaticCheckInterface
     /**
      * Executes static check script, returns output.
      * @param InputInterface $input
-     * @return string
+     * @return void
      */
     public function execute(InputInterface $input);
+
+    /**
+     * Return array containing all errors found after running the execute() function.
+     * @return array
+     */
+    public function getErrors();
+
+    /**
+     * Return string of a short human readable result of the check. For example: "No Dependency errors found."
+     * @return string
+     */
+    public function getOutput();
 }

src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckListInterface.php

@@ -14,7 +14,7 @@ interface StaticCheckListInterface
     /**
      * Gets list of static check script instances
      *
-     * @return \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface[]
+     * @return StaticCheckInterface[]
      */
     public function getStaticChecks();
 }

src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php

@@ -16,14 +16,14 @@ class StaticChecksList implements StaticCheckListInterface
     /**
      * Property contains all static check scripts.
      *
-     * @var \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface[]
+     * @var StaticCheckInterface[]
      */
     private $checks;
 
     /**
      * Constructor
      *
-     * @param array $scripts
+     * @param array $checks
      */
     public function __construct(array $checks = [])
     {

src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php

@@ -6,20 +6,20 @@
 
 namespace Magento\FunctionalTestingFramework\StaticCheck;
 
-use Magento\FunctionalTestingFramework\Config\Data;
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
+use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\Page\Handlers\PageObjectHandler;
 use Magento\FunctionalTestingFramework\Page\Handlers\SectionObjectHandler;
-use Magento\FunctionalTestingFramework\Page\Objects\SectionObject;
 use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Util\ModuleResolver;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Finder\Finder;
+use Exception;
 
 /**
  * Class TestDependencyCheck
@@ -62,11 +62,24 @@ class TestDependencyCheck implements StaticCheckInterface
      */
     private $alreadyExtractedDependencies;
 
+    /**
+     * Array containing all errors found after running the execute() function.
+     * @var array
+     */
+    private $errors;
+
+    /**
+     * String representing the output summary found after running the execute() function.
+     * @var string
+     */
+    private $output;
+
     /**
      * Checks test dependencies, determined by references in tests versus the dependencies listed in the Magento module
      *
      * @param InputInterface $input
      * @return string
+     * @throws Exception;
      * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     public function execute(InputInterface $input)
@@ -98,21 +111,39 @@ public function execute(InputInterface $input)
         $actionGroupXmlFiles = $this->buildFileList($allModules, $filePaths[1]);
         $dataXmlFiles= $this->buildFileList($allModules, $filePaths[2]);
 
-        $testErrors = [];
-        $testErrors += $this->findErrorsInFileSet($testXmlFiles);
-        $testErrors += $this->findErrorsInFileSet($actionGroupXmlFiles);
-        $testErrors += $this->findErrorsInFileSet($dataXmlFiles);
+        $this->errors = [];
+        $this->errors += $this->findErrorsInFileSet($testXmlFiles);
+        $this->errors += $this->findErrorsInFileSet($actionGroupXmlFiles);
+        $this->errors += $this->findErrorsInFileSet($dataXmlFiles);
 
-        //print all errors to file
-        return $this->printErrorsToFile($testErrors);
+        // hold on to the output and print any errors to a file
+        $this->output = $this->printErrorsToFile();
+    }
+
+    /**
+     * Return array containing all errors found after running the execute() function.
+     * @return array
+     */
+    public function getErrors()
+    {
+        return $this->errors;
+    }
+
+    /**
+     * Return string of a short human readable result of the check. For example: "No Dependency errors found."
+     * @return string
+     */
+    public function getOutput()
+    {
+        return $this->output;
     }
 
     /**
      * Finds all reference errors in given set of files
      * @param Finder $files
      * @return array
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
+     * @throws TestReferenceException
+     * @throws XmlException
      */
     private function findErrorsInFileSet($files)
     {
@@ -333,8 +364,7 @@ private function buildFileList($modulePaths, $path)
      * Attempts to find any MFTF entity by its name. Returns null if none are found.
      * @param string $name
      * @return mixed
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
+     * @throws XmlException
      */
     private function findEntity($name)
     {
@@ -363,24 +393,29 @@ private function findEntity($name)
 
     /**
      * Prints out given errors to file, and returns summary result string
-     * @param array $errors
      * @return string
      */
-    private function printErrorsToFile($errors)
+    private function printErrorsToFile()
     {
+        $errors = $this->getErrors();
+
         if (empty($errors)) {
             return "No Dependency errors found.";
         }
+
         $outputPath = getcwd() . DIRECTORY_SEPARATOR . "mftf-dependency-checks.txt";
         $fileResource = fopen($outputPath, 'w');
         $header = "MFTF File Dependency Check:\n";
         fwrite($fileResource, $header);
+
         foreach ($errors as $test => $error) {
             fwrite($fileResource, $error[0] . PHP_EOL);
         }
+
         fclose($fileResource);
         $errorCount = count($errors);
         $output = "Dependency errors found across {$errorCount} file(s). Error details output to {$outputPath}";
+
         return $output;
     }
 }