Merge branch 'develop' of github.com:magento/magento2-functional-testing-framework into waitforpageload-after-amonpage

Author: Cristian101

CHANGELOG.md

@@ -1,6 +1,56 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.2.0
+-----
+### Enhancements  
+* Traceability
+    * Javascript errors are now logged and reported in test output.
+    * Test failures are no longer overwritten by failures in an `<after>` hook.
+    * Tests will no longer execute an `<after>` hook twice if a failure triggered in the `<after>` hook.
+    * Tests marked with `<group value="skip">` will now appear in generated Allure reports.
+        * Along with the above, the `robo group` command no longer omits the `skip` group (skipped tests are picked up but not fully executed).
+* Modularity
+    * MFTF no longer relies on relative pathing to determine its path to tests or Magento (favoring composer information if available).
+    * Tests and test materials are now read in from Magento modules as well as extensions in addition to `dev/tests/acceptance`.
+        * See DevDocs `Getting Started` for details on expected paths and merge order.
+* Customizability
+    * Creation of Suites is now supported
+        * `<suite>` can include tests via `name`, module, or `<group>` tags.
+        * Consolidation of preconditions can be achieved via use of `<before/after>` tags in a `<suite>`
+            * All normal test actions are supported
+            * Data returned from actions is not available for reference in subsequent tests (`createData` or `grab` actions).
+        * `robo generate:tests` generates all suites and tests, and can be given a JSON configuration to generate specific test/suites.
+        * See MFTF Devdocs "Suite" page for more details.
+    * `<deleteData>` may now be called against a `url` instead of a stepKey reference.
+    * `<dragAndDrop>` may now be given an additional `x/y` offset.
+    * `<executeJS>` now returns a variable based on what the executed script returns.
+    * Added `<element>` `type="block"`.
+    * `<page>` elements may now be blank (contain no child sections).
+* Maintainability
+    * `robo generate:tests --config parallel` now accepts a `--lines` argument, for grouping and sorting based on test length.
+    * `robo generate:tests` now checks for:
+        * Duplicate step keys within an `actionGroup`.
+        * Ambiguous or invalid `stepKey` references (in merge files).
+    * `robo generate:tests` now suppresses warnings by default. The command now accepts a `--verbose` flag to show full output including warnings.
+
+### Fixes
+* Exception message for the `<conditionalClick>` action now correctly references the `selector` given.
+* Usage of multiple parameterized elements in a `selector` now correctly resolves all element references.
+* Usage of multiple uniqueness references on the same entity now generate correctly.
+* Persisted entity references are correctly interpolated with `<page>` url of `type="admin"`.
+* Metadata that contains 2 or more params in its `url` now correctly resolve parameters.
+* Arguments can now be passed to `x` and `y` attributes in `actionGroup`.
+* Arguments can now be passed to nested `<assert*>` action elements.
+* The `<seeInField>` action can now be used to assert against empty strings.
+* Empty `<data>` elements within an `<entity>` now generate correctly.
+* Mapping of the `<magentoCLI>` to the custom command has been fixed.
+
+### GitHub Issues/Pull requests:
+* [#89](https://github.com/magento/magento2-functional-testing-framework/pull/89) -- Add ability to use array entities as arguments.
+* [#68](https://github.com/magento/magento2-functional-testing-framework/issues/68) -- Excessive double quotes are being generated in WaitForElementChange method arguments (fixed in [#103](https://github.com/magento/magento2-functional-testing-framework/pull/103))
+* [#31](https://github.com/magento/magento2-functional-testing-framework/issues/31) -- Can't run tests without a store having "default" store code (fixed in [#86](https://github.com/magento/magento2-functional-testing-framework/pull/86))
+
 2.1.2
 -----
 ### Enhancements

bin/static-checks

@@ -8,7 +8,7 @@ set -e
 echo "==============================="
 echo "         CODE SNIFFER"
 echo "==============================="
-vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento
+vendor/bin/phpcs ./src --standard=./dev/tests/static/Magento --ignore=src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
 vendor/bin/phpcs ./dev/tests/unit --standard=./dev/tests/static/Magento
 vendor/bin/phpcs ./dev/tests/verification --standard=./dev/tests/static/Magento --ignore=dev/tests/verification/_generated
 echo ""
@@ -22,7 +22,7 @@ echo ""
 echo "==============================="
 echo "       MESS DETECTOR"
 echo "==============================="
-vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated
+vendor/bin/phpmd ./src text /dev/tests/static/Magento/CodeMessDetector/ruleset.xml --exclude _generated,src/Magento/FunctionalTestingFramework/Group,src/Magento/FunctionalTestingFramework/AcceptanceTester.php
 echo ""
 
 echo "==============================="

bin/static-checks.bat

@@ -5,15 +5,15 @@
 
 @echo off
 @echo ===============================PHP CODE SNIFFER REPORT===============================
-call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento
+call vendor\bin\phpcs .\src --standard=.\dev\tests\static\Magento --ignore=src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
 call vendor\bin\phpcs .\dev\tests\unit --standard=.\dev\tests\static\Magento
 call vendor\bin\phpcs .\dev\tests\verification --standard=.\dev\tests\static\Magento --ignore=dev\tests\verification\_generated
 
 @echo ===============================COPY PASTE DETECTOR REPORT===============================
 call vendor\bin\phpcpd .\src
 
 @echo "===============================PHP MESS DETECTOR REPORT===============================
-vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated
+vendor\bin\phpmd .\src text \dev\tests\static\Magento\CodeMessDetector\ruleset.xml --exclude _generated,src\Magento\FunctionalTestingFramework\Group,src\Magento\FunctionalTestingFramework\AcceptanceTester.php
 
 @echo ===============================MAGENTO COPYRIGHT REPORT===============================
 echo msgbox "INFO:Copyright check currently not run as part of .bat implementation" > "%temp%\popup.vbs"

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.1.2",
+    "version": "2.2.0",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {
@@ -17,7 +17,7 @@
         "flow/jsonpath": ">0.2",
         "fzaninotto/faker": "^1.6",
         "mustache/mustache": "~2.5",
-        "symfony/process": ">=2.7 <3.4",
+        "symfony/process": "^2.8 || ^3.1 || ^4.0",
         "vlucas/phpdotenv": "^2.4"
     },
     "require-dev": {

composer.lock

@@ -16,8 +16,12 @@
     'includePaths' => [PROJECT_ROOT . '/src']
 ]);
 
-// force php to generate
-$GLOBALS['FORCE_PHP_GENERATE'] = true;
+// set mftf appplication context
+\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::create(
+    true,
+    \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::GENERATION_PHASE,
+    true
+);
 
 // Load needed framework env params
 $TEST_ENVS = [
@@ -42,6 +46,7 @@
 
 defined('TESTS_BP') || define('TESTS_BP', __DIR__);
 defined('TESTS_MODULE_PATH') || define('TESTS_MODULE_PATH', TESTS_BP . $RELATIVE_TESTS_MODULE_PATH);
+defined('MAGENTO_BP') || define('MAGENTO_BP', __DIR__);
 
 $utilDir = DIRECTORY_SEPARATOR . 'Util'. DIRECTORY_SEPARATOR . '*.php';
 

dev/tests/_bootstrap.php

@@ -51,6 +51,25 @@ public function testGetPageObject()
         $this->assertNull($invalidPage);
     }
 
+    public function testGetEmptyPage()
+    {
+        $mockData = [
+            "testPage1" => [
+                "url" => "testURL1",
+                "module" => "testModule1",
+                "section" => [
+                ],
+                "area" => "test"
+            ]];
+        $this->setMockParserOutput($mockData);
+
+        // get pages
+        $page = PageObjectHandler::getInstance()->getObject('testPage1');
+
+        // Empty page has been read in and gotten without an exception being thrown.
+        $this->addToAssertionCount(1);
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *

dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php

@@ -0,0 +1,109 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestFramework\Suite\Handlers;
+
+use AspectMock\Test as AspectMock;
+use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager;
+use Magento\FunctionalTestingFramework\ObjectManagerFactory;
+use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
+use Magento\FunctionalTestingFramework\Suite\Parsers\SuiteDataParser;
+use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
+use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
+use PHPUnit\Framework\TestCase;
+use tests\unit\Util\SuiteDataArrayBuilder;
+use tests\unit\Util\TestDataArrayBuilder;
+
+class SuiteObjectHandlerTest extends TestCase
+{
+    /**
+     * Tests basic parsing and accesors of suite object and suite object supporting classes
+     */
+    public function testGetSuiteObject()
+    {
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockData = $suiteDataArrayBuilder
+            ->withName('basicTestSuite')
+            ->withAfterHook()
+            ->withBeforeHook()
+            ->includeTests(['simpleTest'])
+            ->includeGroups(['group1'])
+            ->excludeTests(['group1Test2'])
+            ->excludeGroups(['group2'])
+            ->build();
+
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest = $testDataArrayBuilder
+            ->withName('simpleTest')
+            ->withTestActions()
+            ->build();
+
+        $mockGroup1Test1 = $testDataArrayBuilder
+            ->withName('group1Test1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+
+        $mockGroup1Test2 = $testDataArrayBuilder
+            ->withName('group1Test2')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+
+        $mockGroup2Test1 = $testDataArrayBuilder
+            ->withName('group2Test1')
+            ->withAnnotations(['group' => [['value' => 'group2']]])
+            ->withTestActions()
+            ->build();
+
+        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
+
+        // parse and retrieve suite object with mocked data
+        $basicTestSuiteObj = SuiteObjectHandler::getInstance()->getObject('basicTestSuite');
+
+        // assert on created suite object
+        $this->assertEquals($basicTestSuiteObj->getName(), 'basicTestSuite');
+        $this->assertCount(2, $basicTestSuiteObj->getTests());
+        $this->assertNotEmpty($basicTestSuiteObj->getBeforeHook());
+        $this->assertNotEmpty($basicTestSuiteObj->getAfterHook());
+    }
+
+    /**
+     * Function used to set mock for parser return and force init method to run between tests.
+     *
+     * @param array $testData
+     * @throws \Exception
+     */
+    private function setMockTestAndSuiteParserOutput($testData, $suiteData)
+    {
+        // clear test object handler value to inject parsed content
+        $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
+        $property->setAccessible(true);
+        $property->setValue(null);
+
+        // clear suite object handler value to inject parsed content
+        $property = new \ReflectionProperty(SuiteObjectHandler::class, 'SUITE_OBJECT_HANLDER_INSTANCE');
+        $property->setAccessible(true);
+        $property->setValue(null);
+
+        $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $testData])->make();
+        $mockSuiteDataParser = AspectMock::double(SuiteDataParser::class, ['readSuiteData' => $suiteData])->make();
+        $instance = AspectMock::double(
+            ObjectManager::class,
+            ['create' => function ($clazz) use ($mockDataParser, $mockSuiteDataParser) {
+                if ($clazz == TestDataParser::class) {
+                    return $mockDataParser;
+                }
+
+                if ($clazz == SuiteDataParser::class) {
+                    return $mockSuiteDataParser;
+                }
+            }]
+        )->make();
+        // bypass the private constructor
+        AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]);
+    }
+}

dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php

@@ -0,0 +1,32 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestFramework\Test\Config;
+
+use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
+use Magento\FunctionalTestingFramework\Test\Config\ActionGroupDom;
+use PHPUnit\Framework\TestCase;
+
+class ActionGroupDomTest extends TestCase
+{
+    /**
+     * Test Action Group duplicate step key validation
+     */
+    public function testActionGroupDomStepKeyValidation()
+    {
+        $sampleXml = "<actionGroups>
+            <actionGroup name=\"actionGroupWithoutArguments\">
+                <wait time=\"1\" stepKey=\"waitForNothing\" />
+                <wait time=\"2\" stepKey=\"waitForNothing\" />
+            </actionGroup>
+         </actionGroups>";
+
+        $exceptionCollector = new ExceptionCollector();
+        $actionDom = new ActionGroupDom($sampleXml, 'dupeStepKeyActionGroup.xml', $exceptionCollector);
+
+        $this->expectException(\Exception::class);
+        $exceptionCollector->throwException();
+    }
+}

dev/tests/unit/Magento/FunctionalTestFramework/Test/Config/ActionGroupDomTest.php

@@ -255,6 +255,36 @@ public function testResolveDataInUserInput()
         $this->assertEquals($expected, $actionObject->getCustomActionAttributes());
     }
 
+    /**
+     * {{EntityDataObject.values}} should be replaced with ["value1","value2"]
+     */
+    public function testResolveArrayData()
+    {
+        // Set up mocks
+        $actionObject = new ActionObject('merge123', 'fillField', [
+            'selector' => '#selector',
+            'userInput' => '{{EntityDataObject.values}}'
+        ]);
+        $entityDataObject = new EntityDataObject('EntityDataObject', 'test', [
+            'values' => [
+                'value1',
+                'value2',
+                '"My" Value'
+            ]
+        ], [], '', '');
+        $this->mockDataHandlerWithData($entityDataObject);
+
+        // Call the method under test
+        $actionObject->resolveReferences();
+
+        // Verify
+        $expected = [
+            'selector' => '#selector',
+            'userInput' => '["value1","value2","\"My\" Value"]'
+        ];
+        $this->assertEquals($expected, $actionObject->getCustomActionAttributes());
+    }
+
     /**
      * Action object should throw an exception if a reference to a parameterized selector has too few given args.
      */