From 75b6b88527869d8b3034d6ea3f26bf811da8d983 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 27 Aug 2019 09:48:35 -0500 Subject: [PATCH 1/6] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual & Expected Results in Allure - Added Allure firing of attachment to steps --- .../Module/MagentoWebDriver.php | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index d268e23b4..0c8f632b3 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -9,6 +9,7 @@ use Codeception\Module\WebDriver; use Codeception\Test\Descriptor; use Codeception\TestInterface; +use Yandex\Allure\Adapter\Allure; use Facebook\WebDriver\Interactions\WebDriverActions; use Codeception\Exception\ModuleConfigException; use Codeception\Exception\ModuleException; @@ -18,6 +19,8 @@ use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; +use Yandex\Allure\Adapter\Event\AddAttachmentEvent; +use Yandex\Allure\Adapter\Event\AddParameterEvent; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -198,7 +201,10 @@ public function _getCurrentUri() */ public function dontSeeCurrentUrlEquals($url) { - $this->assertNotEquals($url, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $url\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotEquals($url, $actualUrl); } /** @@ -209,7 +215,10 @@ public function dontSeeCurrentUrlEquals($url) */ public function dontSeeCurrentUrlMatches($regex) { - $this->assertNotRegExp($regex, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $regex\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotRegExp($regex, $actualUrl); } /** @@ -220,7 +229,10 @@ public function dontSeeCurrentUrlMatches($regex) */ public function dontSeeInCurrentUrl($needle) { - $this->assertNotContains($needle, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $needle\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotContains($needle, $actualUrl); } /** @@ -254,7 +266,10 @@ public function grabFromCurrentUrl($regex = null) */ public function seeCurrentUrlEquals($url) { - $this->assertEquals($url, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $url\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertEquals($url, $actualUrl); } /** @@ -265,7 +280,10 @@ public function seeCurrentUrlEquals($url) */ public function seeCurrentUrlMatches($regex) { - $this->assertRegExp($regex, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $regex\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertRegExp($regex, $url); } /** @@ -276,7 +294,10 @@ public function seeCurrentUrlMatches($regex) */ public function seeInCurrentUrl($needle) { - $this->assertContains($needle, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $needle\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertContains($needle, $actualUrl); } /** From beec38835e3e38b60e881bc7a504b980984b2d5d Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 28 Aug 2019 13:12:09 -0500 Subject: [PATCH 2/6] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual & Expected Results in Allure - bugfix for variable name --- .../FunctionalTestingFramework/Module/MagentoWebDriver.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 0c8f632b3..830fd40c4 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -283,7 +283,7 @@ public function seeCurrentUrlMatches($regex) $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); - $this->assertRegExp($regex, $url); + $this->assertRegExp($regex, $actualUrl); } /** From 300619dcccefaaf83ccb1f3f5efce8273c43a5aa Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 29 Aug 2019 14:22:28 -0500 Subject: [PATCH 3/6] MQE-1633: Add API Request To Output for MFTF API Actions - Added AllureHelper + Unit tests - Added attachment event to curlHandler classes --- .../Allure/AllureHelperTest.php | 97 +++++++++++++++++++ .../Allure/AllureHelper.php | 39 ++++++++ .../DataGenerator/Persist/CurlHandler.php | 9 ++ 3 files changed, 145 insertions(+) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php create mode 100644 src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php new file mode 100644 index 000000000..048c6f7de --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php @@ -0,0 +1,97 @@ +mockAttachmentWriteEvent(); + $expectedData = "string"; + $expectedCaption = "caption"; + + //Prepare Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('firstStep')); + + //Call function + AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption); + + // Assert Attachment is created as expected + $step = Allure::lifecycle()->getStepStorage()->pollLast(); + $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null); + $this->assertEquals($step->getAttachments()[0], $expectedAttachment); + } + + /** + * AddAttachmentToLastStep should add an attachment only to the last step + * @throws \Yandex\Allure\Adapter\AllureException + */ + public function testAddAttachmentToLastStep() + { + $this->mockAttachmentWriteEvent(); + $expectedData = "string"; + $expectedCaption = "caption"; + + //Prepare Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('firstStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('firstStep')); + Allure::lifecycle()->fire(new StepStartedEvent('secondStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('secondStep')); + + //Call function + AllureHelper::addAttachmentToLastStep($expectedData, $expectedCaption); + + //Continue Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('thirdStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('thirdStep')); + + // Assert Attachment is created as expected on the right step + $rootStep = Allure::lifecycle()->getStepStorage()->pollLast(); + + $firstStep = $rootStep->getSteps()[0]; + $secondStep = $rootStep->getSteps()[1]; + $thirdStep = $rootStep->getSteps()[2]; + + $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null); + $this->assertEmpty($firstStep->getAttachments()); + $this->assertEquals($secondStep->getAttachments()[0], $expectedAttachment); + $this->assertEmpty($thirdStep->getAttachments()); + } + + /** + * Mock file system manipulation function + * @throws \Exception + */ + public function mockAttachmentWriteEvent() + { + AspectMock::double(AddAttachmentEvent::class, [ + "getAttachmentFileName" => self::MOCK_FILENAME + ]); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php new file mode 100644 index 000000000..232c3387a --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php @@ -0,0 +1,39 @@ +fire(new AddAttachmentEvent($data, $caption)); + } + + /** + * Adds Attachment to the last executed step. Required due to Allure root-step behavior + * @param mixed $data + * @param string $caption + * @return void + */ + public static function addAttachmentToLastStep($data, $caption) + { + $rootStep = Allure::lifecycle()->getStepStorage()->getLast(); + $trueLastStep = array_last($rootStep->getSteps()); + + $attachmentEvent = new AddAttachmentEvent($data, $caption); + $attachmentEvent->process($trueLastStep); + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index f6d8773bc..4b184c206 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\DataGenerator\Persist; +use Magento\FunctionalTestingFramework\Allure\AllureHelper; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\AdminExecutor; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\FrontendExecutor; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\WebapiExecutor; @@ -166,6 +167,14 @@ public function executeRequest($dependentEntities) $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); + AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); + AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); + AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body'); + AllureHelper::addAttachmentToLastStep( + json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES), + 'Response Data' + ); + return $response; } From fdcda9a93cbeb243d90f2384a9bbe02959140ef9 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 29 Aug 2019 14:26:36 -0500 Subject: [PATCH 4/6] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual - Added contents of 1633, changed the calls to use Helper class instead --- .../Module/MagentoWebDriver.php | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 830fd40c4..731643eaf 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -9,7 +9,7 @@ use Codeception\Module\WebDriver; use Codeception\Test\Descriptor; use Codeception\TestInterface; -use Yandex\Allure\Adapter\Allure; +use Magento\FunctionalTestingFramework\Allure\AllureHelper; use Facebook\WebDriver\Interactions\WebDriverActions; use Codeception\Exception\ModuleConfigException; use Codeception\Exception\ModuleException; @@ -19,8 +19,6 @@ use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; -use Yandex\Allure\Adapter\Event\AddAttachmentEvent; -use Yandex\Allure\Adapter\Event\AddParameterEvent; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -203,7 +201,7 @@ public function dontSeeCurrentUrlEquals($url) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $url\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotEquals($url, $actualUrl); } @@ -217,7 +215,7 @@ public function dontSeeCurrentUrlMatches($regex) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotRegExp($regex, $actualUrl); } @@ -231,7 +229,7 @@ public function dontSeeInCurrentUrl($needle) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $needle\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotContains($needle, $actualUrl); } @@ -268,7 +266,7 @@ public function seeCurrentUrlEquals($url) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $url\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertEquals($url, $actualUrl); } @@ -282,7 +280,7 @@ public function seeCurrentUrlMatches($regex) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertRegExp($regex, $actualUrl); } @@ -296,7 +294,7 @@ public function seeInCurrentUrl($needle) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $needle\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertContains($needle, $actualUrl); } From 0dd8840fd7f2029098b9caef6e33dd098093ab7e Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 30 Aug 2019 09:54:00 -0500 Subject: [PATCH 5/6] MQE-1633: Add API Request To Output for MFTF API Actions - Unit test fix --- dev/tests/_bootstrap.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index e06e373d4..b41f80394 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -17,7 +17,10 @@ $kernel = \AspectMock\Kernel::getInstance(); $kernel->init([ 'debug' => true, - 'includePaths' => [PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src'], + 'includePaths' => [ + PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src', + PROJECT_ROOT . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'allure-framework' + ], 'cacheDir' => PROJECT_ROOT . DIRECTORY_SEPARATOR . 'dev' . From fdd5ec09e5c94f0cb874b6cb207ab6da6cceb0fa Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 30 Aug 2019 10:17:41 -0500 Subject: [PATCH 6/6] MQE-1633: Add API Request To Output for MFTF API Actions - Updated docblock --- src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php index 232c3387a..d8f1c63d8 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php +++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php @@ -23,7 +23,8 @@ public static function addAttachmentToCurrentStep($data, $caption) } /** - * Adds Attachment to the last executed step. Required due to Allure root-step behavior + * Adds Attachment to the last executed step. + * Use this when adding attachments outside of an $I->doSomething() step/context. * @param mixed $data * @param string $caption * @return void