MQE-1884: MFTF - <after> failures override other failures

composer.json

@@ -12,7 +12,7 @@
         "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0",
         "ext-curl": "*",
         "allure-framework/allure-codeception": "~1.3.0",
-        "codeception/codeception": "~2.3.4 || ~2.4.0 ",
+        "codeception/codeception": "~2.4.5",
         "composer/composer": "^1.6",
         "consolidation/robo": "^1.0.0",
         "csharpru/vault-php": "~3.5.3",

src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php

@@ -13,11 +13,11 @@
 /**
  * Class TestContextExtension
  * @SuppressWarnings(PHPMD.UnusedPrivateField)
+ * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
  */
 class TestContextExtension extends BaseExtension
 {
     const TEST_PHASE_AFTER = "_after";
-    const CODECEPT_AFTER_VERSION = "2.3.9";
     const TEST_FAILED_FILE = 'failed';
 
     /**
@@ -36,7 +36,6 @@ public function _initialize()
     {
         $events = [
             Events::TEST_START => 'testStart',
-            Events::TEST_FAIL => 'testFail',
             Events::STEP_AFTER => 'afterStep',
             Events::TEST_END => 'testEnd',
             Events::RESULT_PRINT_AFTER => 'saveFailed'
@@ -57,23 +56,7 @@ public function testStart()
     }
 
     /**
-     * Codeception event listener function, triggered on test failure.
-     * @param \Codeception\Event\FailEvent $e
-     * @return void
-     */
-    public function testFail(\Codeception\Event\FailEvent $e)
-    {
-        $cest = $e->getTest();
-        $context = $this->extractContext($e->getFail()->getTrace(), $cest->getTestMethod());
-        // Do not attempt to run _after if failure was in the _after block
-        // Try to run _after but catch exceptions to prevent them from overwriting original failure.
-        if ($context != TestContextExtension::TEST_PHASE_AFTER) {
-            $this->runAfterBlock($e, $cest);
-        }
-    }
-
-    /**
-     * Codeception event listener function, triggered on test ending (naturally or by error).
+     * Codeception event listener function, triggered on test ending naturally or by errors/failures.
      * @param \Codeception\Event\TestEvent $e
      * @return void
      * @throws \Exception
@@ -82,55 +65,33 @@ public function testEnd(\Codeception\Event\TestEvent $e)
     {
         $cest = $e->getTest();
 
-        //Access private TestResultObject to find stack and if there are any errors (as opposed to failures)
+        //Access private TestResultObject to find stack and if there are any errors/failures
         $testResultObject = call_user_func(\Closure::bind(
             function () use ($cest) {
                 return $cest->getTestResultObject();
             },
             $cest
         ));
-        $errors = $testResultObject->errors();
-        if (!empty($errors)) {
-            foreach ($errors as $error) {
-                if ($error->failedTest()->getTestMethod() == $cest->getName()) {
-                    $stack = $errors[0]->thrownException()->getTrace();
-                    $context = $this->extractContext($stack, $cest->getTestMethod());
-                    // Do not attempt to run _after if failure was in the _after block
-                    // Try to run _after but catch exceptions to prevent them from overwriting original failure.
-                    if ($context != TestContextExtension::TEST_PHASE_AFTER) {
-                        $this->runAfterBlock($e, $cest);
-                    }
-                    continue;
+
+        // check for errors in all test hooks and attach in allure
+        if (!empty($testResultObject->errors())) {
+            foreach ($testResultObject->errors() as $error) {
+                if ($error->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                    $this->attachExceptionToAllure($error->thrownException(), $cest->getTestMethod());
                 }
             }
         }
-        // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true
-        $this->getDriver()->_runAfter($e->getTest());
-    }
 
-    /**
-     * Runs cest's after block, if necessary.
-     * @param \Symfony\Component\EventDispatcher\Event $e
-     * @param \Codeception\TestInterface               $cest
-     * @return void
-     */
-    private function runAfterBlock($e, $cest)
-    {
-        try {
-            $actorClass = $e->getTest()->getMetadata()->getCurrent('actor');
-            $I = new $actorClass($cest->getScenario());
-            if (version_compare(\Codeception\Codecept::VERSION, TestContextExtension::CODECEPT_AFTER_VERSION, "<=")) {
-                call_user_func(\Closure::bind(
-                    function () use ($cest, $I) {
-                        $cest->executeHook($I, 'after');
-                    },
-                    null,
-                    $cest
-                ));
+        // check for failures in all test hooks and attach in allure
+        if (!empty($testResultObject->failures())) {
+            foreach ($testResultObject->failures() as $failure) {
+                if ($failure->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                    $this->attachExceptionToAllure($failure->thrownException(), $cest->getTestMethod());
+                }
             }
-        } catch (\Exception $e) {
-            // Do not rethrow Exception
         }
+        // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true
+        $this->getDriver()->_runAfter($e->getTest());
     }
 
     /**
@@ -150,6 +111,40 @@ public function extractContext($trace, $class)
         return null;
     }
 
+    /**
+     * Attach stack trace of exceptions thrown in each test hook to allure.
+     * @param  \Exception $exception
+     * @param  string     $testMethod
+     * @return mixed
+     */
+    public function attachExceptionToAllure($exception, $testMethod)
+    {
+        if (is_subclass_of($exception, \PHPUnit\Framework\Exception::class)) {
+            $trace = $exception->getSerializableTrace();
+        } else {
+            $trace = $exception->getTrace();
+        }
+
+        $context = $this->extractContext($trace, $testMethod);
+
+        AllureHelper::addAttachmentToCurrentStep($exception, $context . 'Exception');
+
+        //pop suppressed exceptions and attach to allure
+        $change = function () {
+            if ($this instanceof \PHPUnit\Framework\ExceptionWrapper) {
+                return $this->previous;
+            } else {
+                return $this->getPrevious();
+            }
+        };
+
+        $previousException = $change->call($exception);
+
+        if ($previousException !== null) {
+            $this->attachExceptionToAllure($previousException, $testMethod);
+        }
+    }
+
     /**
      * Codeception event listener function, triggered before step.
      * Check if it's a new page.