From 83dab03624a7f4df6896e6faadc9cccd63df213c Mon Sep 17 00:00:00 2001 From: Ian Meron Date: Wed, 18 Apr 2018 09:57:30 -0500 Subject: [PATCH 1/3] MQE-957: MFTF Changes to support DEVOPS-2029 - add new skipped logic to TestObject - update Group Object file generation to include try/catch --- .../Allure/Adapter/MagentoAllureAdapter.php | 29 ++++++++++++++++++- .../Suite/views/partials/testActions.mustache | 10 +++++-- .../Test/Objects/TestObject.php | 10 ++++++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php index c10a04506..393f29008 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php +++ b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Allure\Adapter; use Magento\FunctionalTestingFramework\Data\Argument\Interpreter\NullType; +use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Yandex\Allure\Adapter\AllureAdapter; use Codeception\Event\SuiteEvent; @@ -44,7 +45,7 @@ public function suiteBefore(SuiteEvent $suiteEvent) if ($this->getGroup() != null) { $suite = $suiteEvent->getSuite(); - $suiteName = ($suite->getName()) . "\\" . $this->getGroup(); + $suiteName = ($suite->getName()) . "\\" . $this->sanitizeGroupName($this->getGroup()); call_user_func(\Closure::bind( function () use ($suite, $suiteName) { @@ -64,4 +65,30 @@ function () use ($suite, $suiteName) { // call parent function parent::suiteBefore($changeSuiteEvent); } + + /** + * Function which santizes any group names changed by the framework for execution in order to consolidate reporting. + * + * @param string $group + * @return string + */ + private function sanitizeGroupName($group) + { + $suiteNames = array_keys(SuiteObjectHandler::getInstance()->getAllObjects()); + $exactMatch = in_array($group, $suiteNames); + + // if this is an existing suite name we dont' need to worry about changing it + if ($exactMatch || strpos($group, "_") === false) { + return $group; + } + + // if we can't find this group in the generated suites we have to assume that the group was split for generation + $groupNameSplit = explode("_", $group); + array_pop($groupNameSplit); + $originalName = implode("_", $groupNameSplit); + + // confirm our original name is one of the existing suite names otherwise just return the original group name + $originalName = in_array($originalName, $suiteNames) ? $originalName : $group; + return $originalName; + } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index 2f6c41ef4..33dadc1d7 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -10,17 +10,21 @@ if ($webDriver->webDriver != null) { // initialize the webdriver session $webDriver->_initializeSession(); - -// execute user specified actions +try { + // execute user specified actions {{/webDriverInit}} {{#webDriverReset}} +} catch (\Exception $e) { + print $e->getMessage(); +} + // reset configuration and close session $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); $webDriver->webDriver->close(); $webDriver->webDriver = null; {{/webDriverReset}} {{#action}} -{{{action}}} + {{{action}}} {{/action}} {{#createData}} ${{entityName}} = DataObjectHandler::getInstance()->getObject("{{entityName}}"); diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php index b2e4ee907..8d8700868 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php @@ -132,6 +132,10 @@ public function getAnnotations() */ public function getHooks() { + // if this test is skipped we do not want any before/after actions to generate as the tests will not run + if ($this->isSkipped()) { + return []; + } return $this->hooks; } @@ -142,6 +146,11 @@ public function getHooks() */ public function getTestActionCount() { + // a skipped action results in a single skip being appended to the beginning of the test and no execution + if ($this->isSkipped()) { + return 1; + } + $hookActions = 0; if (array_key_exists('before', $this->hooks)) { $hookActions += count($this->hooks['before']->getActions()); @@ -152,7 +161,6 @@ public function getTestActionCount() } $testActions = count($this->getOrderedActions()); - return $hookActions + $testActions; } From 32770342ab927b31c0ed48a9c23ebc1426058d95 Mon Sep 17 00:00:00 2001 From: Ian Meron Date: Wed, 18 Apr 2018 13:58:22 -0500 Subject: [PATCH 2/3] MQE-957: MFTF Changes to support DEVOPS-2029 - fix for try catch not capturing delete/create data --- .../Suite/views/SuiteClass.mustache | 13 +++++++++++-- .../Suite/views/partials/testActions.mustache | 7 +------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index a1eac08fc..b6051912d 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -34,7 +34,11 @@ class {{suiteName}} extends \Codeception\GroupObject if (self::$CURRENT_TEST_RUN == 1) { print sprintf(self::$HOOK_EXECUTION_INIT, "before"); - {{> testActions}} + try { + {{> testActions}} + } catch (\Exception $e) { + print $e->getMessage(); + } print sprintf(self::$HOOK_EXECUTION_END, "before"); } @@ -47,7 +51,12 @@ class {{suiteName}} extends \Codeception\GroupObject if (self::$CURRENT_TEST_RUN == self::$TEST_COUNT) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); - {{> testActions}} + try { + {{> testActions}} + } catch (\Exception $e) { + print $e->getMessage(); + } + print sprintf(self::$HOOK_EXECUTION_END, "after"); } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index 33dadc1d7..46951aa46 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -10,13 +10,8 @@ if ($webDriver->webDriver != null) { // initialize the webdriver session $webDriver->_initializeSession(); -try { - // execute user specified actions {{/webDriverInit}} {{#webDriverReset}} -} catch (\Exception $e) { - print $e->getMessage(); -} // reset configuration and close session $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); @@ -24,7 +19,7 @@ $webDriver->webDriver->close(); $webDriver->webDriver = null; {{/webDriverReset}} {{#action}} - {{{action}}} +{{{action}}} {{/action}} {{#createData}} ${{entityName}} = DataObjectHandler::getInstance()->getObject("{{entityName}}"); From e4c1f25841704a0b1aa137f8eba76303dbd43492 Mon Sep 17 00:00:00 2001 From: Ian Meron Date: Thu, 19 Apr 2018 10:29:16 -0500 Subject: [PATCH 3/3] MQE-957: MFTF Changes to support DEVOPS-2029 - mark tests as skipped if we see precondition failure --- .../Suite/views/SuiteClass.mustache | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index b6051912d..cee281f36 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -17,8 +17,9 @@ use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHand class {{suiteName}} extends \Codeception\GroupObject { public static $group = '{{suiteName}}'; - private static $TEST_COUNT = {{testCount}}; - private static $CURRENT_TEST_RUN = 0; + private $testCount = {{testCount}}; + private $preconditionFailure = null; + private $currentTestRun = 0; private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of {{suiteName}} suite %s block ********/\n"; private static $HOOK_EXECUTION_END = "\n/******** Execution of {{suiteName}} suite %s block complete ********/\n"; {{#var}} @@ -29,15 +30,25 @@ class {{suiteName}} extends \Codeception\GroupObject public function _before(\Codeception\Event\TestEvent $e) { // increment test count per execution - self::$CURRENT_TEST_RUN++; + $this->currentTestRun++; + $this->executePreConditions(); - if (self::$CURRENT_TEST_RUN == 1) { + if ($this->preconditionFailure != null) { + //if our preconditions fail, we need to mark all the tests as incomplete. + $e->getTest()->getMetadata()->setIncomplete($this->preconditionFailure); + } + } + + + private function executePreConditions() + { + if ($this->currentTestRun == 1) { print sprintf(self::$HOOK_EXECUTION_INIT, "before"); try { {{> testActions}} - } catch (\Exception $e) { - print $e->getMessage(); + } catch (\Exception $exception) { + $this->preconditionFailure = $exception->getMessage(); } print sprintf(self::$HOOK_EXECUTION_END, "before"); @@ -48,16 +59,21 @@ class {{suiteName}} extends \Codeception\GroupObject {{#after}} public function _after(\Codeception\Event\TestEvent $e) { - if (self::$CURRENT_TEST_RUN == self::$TEST_COUNT) { + $this->executePostConditions(); + } + + + private function executePostConditions() + { + if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); try { {{> testActions}} - } catch (\Exception $e) { - print $e->getMessage(); + } catch (\Exception $exception) { + print $exception->getMessage(); } - print sprintf(self::$HOOK_EXECUTION_END, "after"); } }