Merge branch 'develop' into MQE-1685

Author: jilu1

dev/tests/_bootstrap.php

@@ -17,7 +17,10 @@
 $kernel = \AspectMock\Kernel::getInstance();
 $kernel->init([
     'debug' => true,
-    'includePaths' => [PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src'],
+    'includePaths' => [
+        PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src',
+        PROJECT_ROOT . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'allure-framework'
+    ],
     'cacheDir' => PROJECT_ROOT .
         DIRECTORY_SEPARATOR .
         'dev' .

dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php

@@ -0,0 +1,97 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestingFramework\Allure;
+
+use Magento\FunctionalTestingFramework\Allure\AllureHelper;
+use Yandex\Allure\Adapter\Allure;
+use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
+use Yandex\Allure\Adapter\Event\StepFinishedEvent;
+use Yandex\Allure\Adapter\Event\StepStartedEvent;
+use Yandex\Allure\Adapter\Model\Attachment;
+use AspectMock\Test as AspectMock;
+use PHPUnit\Framework\TestCase;
+
+class AllureHelperTest extends TestCase
+{
+    const MOCK_FILENAME = 'filename';
+
+    /**
+     * Clear Allure Lifecycle
+     */
+    public function tearDown()
+    {
+        Allure::setDefaultLifecycle();
+    }
+
+    /**
+     * AddAtachmentToStep should add an attachment to the current step
+     * @throws \Yandex\Allure\Adapter\AllureException
+     */
+    public function testAddAttachmentToStep()
+    {
+        $this->mockAttachmentWriteEvent();
+        $expectedData = "string";
+        $expectedCaption = "caption";
+
+        //Prepare Allure lifecycle
+        Allure::lifecycle()->fire(new StepStartedEvent('firstStep'));
+
+        //Call function
+        AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption);
+
+        // Assert Attachment is created as expected
+        $step = Allure::lifecycle()->getStepStorage()->pollLast();
+        $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null);
+        $this->assertEquals($step->getAttachments()[0], $expectedAttachment);
+    }
+
+    /**
+     * AddAttachmentToLastStep should add an attachment only to the last step
+     * @throws \Yandex\Allure\Adapter\AllureException
+     */
+    public function testAddAttachmentToLastStep()
+    {
+        $this->mockAttachmentWriteEvent();
+        $expectedData = "string";
+        $expectedCaption = "caption";
+
+        //Prepare Allure lifecycle
+        Allure::lifecycle()->fire(new StepStartedEvent('firstStep'));
+        Allure::lifecycle()->fire(new StepFinishedEvent('firstStep'));
+        Allure::lifecycle()->fire(new StepStartedEvent('secondStep'));
+        Allure::lifecycle()->fire(new StepFinishedEvent('secondStep'));
+
+        //Call function
+        AllureHelper::addAttachmentToLastStep($expectedData, $expectedCaption);
+
+        //Continue Allure lifecycle
+        Allure::lifecycle()->fire(new StepStartedEvent('thirdStep'));
+        Allure::lifecycle()->fire(new StepFinishedEvent('thirdStep'));
+
+        // Assert Attachment is created as expected on the right step
+        $rootStep = Allure::lifecycle()->getStepStorage()->pollLast();
+
+        $firstStep = $rootStep->getSteps()[0];
+        $secondStep = $rootStep->getSteps()[1];
+        $thirdStep = $rootStep->getSteps()[2];
+
+        $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null);
+        $this->assertEmpty($firstStep->getAttachments());
+        $this->assertEquals($secondStep->getAttachments()[0], $expectedAttachment);
+        $this->assertEmpty($thirdStep->getAttachments());
+    }
+
+    /**
+     * Mock file system manipulation function
+     * @throws \Exception
+     */
+    public function mockAttachmentWriteEvent()
+    {
+        AspectMock::double(AddAttachmentEvent::class, [
+            "getAttachmentFileName" => self::MOCK_FILENAME
+        ]);
+    }
+}

docs/commands/mftf.md

@@ -58,6 +58,14 @@ vendor/bin/mftf run:test LoginAsAdminTest LoginAsCustomerTest -r
 
 This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests.
 
+### Generate and run a testManifest.txt file
+
+```bash
+vendor/bin/mftf run:manifest path/to/your/testManifest.txt
+```
+
+This command runs all tests specified in a `testManifest.txt` file. When you generate tests, a `testManifest.txt` file is also generated for you. You can pass this file directly to the `run:manifest` command and it will execute all listed tests. You can also create your own file in the same format to execute a subset of tests. Note: This command does not generate tests.
+
 ### Generate and run previously failed tests
 
 ```bash
@@ -338,6 +346,30 @@ Generate the `LoginCustomerTest` and `StorefrontCreateCustomerTest` tests from X
 vendor/bin/mftf run:test LoginCustomerTest StorefrontCreateCustomerTest
 ```
 
+### `run:manifest`
+
+Runs a testManifest.txt file.
+
+This command runs all tests specified in a testManifest.xml file. It does not generate tests for you. You must do that as first. 
+
+#### Usage
+
+```bash
+vendor/bin/mftf run:manifest path/to/your/testManifest.txt
+```
+
+#### Example testManifest.xml file
+
+Each line should contain either: one test path or one group (-g) reference.
+
+```
+tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php
+-g PaypalTestSuite
+tests/functional/tests/MFTF/_generated/default/SomeOtherTestCest.php
+tests/functional/tests/MFTF/_generated/default/ThirdTestCest.php
+-g SomeOtherSuite
+```
+
 ### `run:failed`
 
 Regenerates and reruns tests that previously failed.

docs/reporting.md

@@ -18,7 +18,7 @@ The `mftf` tool logs output continuously to the `dev/tests/acceptance/mftf.log`
 
 ## Command line
 
-The MFTF reports about its progress during test run when you run the `mftf` CLI tool with [`run:test`][] or [`run:group`][] commands.
+MFTF reports about its progress during test run when you run the `mftf` CLI tool with [`run:test`][] or [`run:group`][] commands.
 
 The report can contain three main parts:
 
@@ -35,14 +35,7 @@ The following sections demonstrate an example interpretation of a complete log s
 
 ### Pre-run check report
 
-First, the MFTF reports about issues with environment.
-In our case, there is an issue with PHP library loading.
-
-```terminal
-PHP Warning:  PHP Startup: Unable to load dynamic library '/usr/local/lib/php/pecl/20160303/php_intl.dll' - dlopen(/usr/local/lib/php/pecl/20160303/php_intl.dll, 9): image not found in Unknown on line 0
-```
-
-Next, the MFTF returns `DEPRECATION` reports alerting you that required test components are missing in XML test definitions.
+First, MFTF returns `DEPRECATION` warnings alerting you that required test components are missing in XML test definitions.
 
 ```terminal
 DEPRECATION: Test AdminFilteringCategoryProductsUsingScopeSelectorTest is missing required annotations.{"testName":"AdminFilteringCategoryProductsUsingScopeSelectorTest","missingAnnotations":"stories"}
@@ -51,7 +44,7 @@ DEPRECATION: Test AdminRemoveProductWeeeAttributeOptionTest is missing required
 Generate Tests Command Run
 ```
 
-`Generate Tests Command Run` indicates the moment when the MFTF has run the tests generation command actually.
+`Generate Tests Command Run` indicates that test generation is finished and tests are able to be executed.
 
 ### Test execution report
 
@@ -72,7 +65,7 @@ Magento\FunctionalTestingFramework.functional Tests (2) ------------------------
 Modules: \Magento\FunctionalTestingFramework\Module\MagentoWebDriver, \Magento\FunctionalTestingFramework\Helper\Acceptance, \Magento\FunctionalTestingFramework\Helper\MagentoFakerData, \Magento\FunctionalTestingFramework\Module\MagentoRestDriver, PhpBrowser, \Magento\FunctionalTestingFramework\Module\MagentoSequence, \Magento\FunctionalTes
 ```
 
-After the test generation command (mentioned in the previous section), the MFTF delegates control to the `vendor/codeception` tool, which is the `Codeception PHP Testing Framework` of version `2.3.9` that uses `PHPUnit` of version `6.5.13`.
+After the test generation command (mentioned in the previous section), MFTF delegates control to the `vendor/codeception` tool, which is the `Codeception PHP Testing Framework` of version `2.3.9` that uses `PHPUnit` of version `6.5.13`.
 
 The tool runs `2 Tests` using the configuration defined in the `functional` suite under the `Magento\FunctionalTestingFramework` namespace.
 The corresponding configuration file is `acceptance/tests/functional.suite.yml`.
@@ -154,7 +147,7 @@ I save screenshot
 FAIL
 ```
 
-When a test step fails, the MFTF always saves a screenshot of the web page with the failing state immediately after the failure occurs.
+When a test step fails, MFTF always saves a screenshot of the web page with the failing state immediately after the failure occurs.
 `I save screenshot` follows the failing test step `I see "#something"` in our case.
 
 A screenshot of the fail goes at the `acceptance/tests/_output` directory in both PNG and HTML formats:
@@ -176,7 +169,7 @@ Actions after `FAIL` are run as a part of the [`after`][] hook of the test.
 
 ### Test result report
 
-After the MFTF completed test execution, it generates a general report about test results along with detailed information about each fail.
+After MFTF completed test execution, it generates a general report about test results along with detailed information about each fail.
 
 ```terminal
 --------------------------------------------------------------------------------
@@ -192,7 +185,7 @@ First you see warnings and deprecations.
 The `DEPRECATION` here is thrown by an MFTF dependency (Symfony) that is out of the scope for test writers and should be considered by MFTF contributors.
 If you encounter this type of reporting, [report an issue][].
 
-Then, the MFTF reports that the test run took 52.43 seconds using 16 MB of system RAM.
+Then, MFTF reports that the test run took 52.43 seconds using 16 MB of system RAM.
 And, finally, that there was `1 failure`.
 
 Next, the report provides details about the test failure.
@@ -268,19 +261,25 @@ FAILURES!
 Tests: 2, Assertions: 3, Failures: 1.
 ```
 
-The MFTF encountered failures due to the last test run, that included *2* tests with *3* assertions.
+MFTF encountered failures due to the last test run, that included *2* tests with *3* assertions.
 *1* assertion fails.
 
 ## Allure
 
-Each time you run tests, the MFTF appends an XML file with results at the `tests/_output/allure-results/` directory.
+Each time you run tests, MFTF appends an XML file with results at the `tests/_output/allure-results/` directory.
 
 The official [Allure Test Report][] documentation is well-covered, so we'll list only the CLI commands that you would need for your day-to-day work.
 
 <div class="bs-callout bs-callout-info">
 The following commands are relative to the Magento installation directory.
 </div>
 
+To generate the HTML Allure report in a temporary folder and open the report in your default web browser:
+
+```bash
+allure serve dev/tests/acceptance/tests/_output/allure-results/
+```
+
 To generate a report to the `allure-report/` at the current directory:
 
 ```bash
@@ -334,12 +333,6 @@ To clean up existing reports before generation (for example after getting new re
 allure generate dev/tests/acceptance/tests/_output/allure-result --clean
 ```
 
-To generate the HTML Allure report in a temporary folder and open the report in your default web browser:
-
-```bash
-allure serve dev/tests/acceptance/tests/_output/allure-results/
-```
-
 Refer to the [Reporting section][] for more Allure CLI details.
 
 <!-- Link definitions -->

src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Magento\FunctionalTestingFramework\Allure;
+
+use Yandex\Allure\Adapter\Allure;
+use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
+
+class AllureHelper
+{
+    /**
+     * Adds attachment to the current step
+     * @param mixed  $data
+     * @param string $caption
+     * @throws \Yandex\Allure\Adapter\AllureException
+     * @return void
+     */
+    public static function addAttachmentToCurrentStep($data, $caption)
+    {
+        Allure::lifecycle()->fire(new AddAttachmentEvent($data, $caption));
+    }
+
+    /**
+     * Adds Attachment to the last executed step.
+     * Use this when adding attachments outside of an $I->doSomething() step/context.
+     * @param mixed  $data
+     * @param string $caption
+     * @return void
+     */
+    public static function addAttachmentToLastStep($data, $caption)
+    {
+        $rootStep = Allure::lifecycle()->getStepStorage()->getLast();
+        $trueLastStep = array_last($rootStep->getSteps());
+
+        $attachmentEvent = new AddAttachmentEvent($data, $caption);
+        $attachmentEvent->process($trueLastStep);
+    }
+}

src/Magento/FunctionalTestingFramework/Console/CommandList.php

@@ -37,6 +37,7 @@ public function __construct(array $commands = [])
              'run:test' => new RunTestCommand(),
              'run:group' => new RunTestGroupCommand(),
              'run:failed' => new RunTestFailedCommand(),
+             'run:manifest' => new RunManifestCommand(),
              'setup:env' => new SetupEnvCommand(),
              'upgrade:tests' => new UpgradeTestsCommand(),
              'generate:docs' => new GenerateDocsCommand(),