Skip to content

Commit f80e2db

Browse files
jilu1jilu1
committed
MQE-2494: Missing steps in Allure logs for certain test failures
1 parent 4640c03 commit f80e2db

File tree

6 files changed

+120
-18
lines changed

6 files changed

+120
-18
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ public function testSortWithSuites()
101101
$this->assertCount(5, $actualResult);
102102

103103
$expectedResults = [
104-
1 => ['mockSuite1_0'],
105-
2 => ['mockSuite1_1'],
104+
1 => ['mockSuite1_0_G'],
105+
2 => ['mockSuite1_1_G'],
106106
3 => ['test3'],
107107
4 => ['test2','test5', 'test4'],
108108
5 => ['test1'],

‎dev/tests/verification/Tests/SuiteGenerationTest.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,10 +118,10 @@ public function testSuiteGenerationParallel()
118118
$groupName = 'functionalSuite1';
119119

120120
$expectedGroups = [
121-
'functionalSuite1_0',
122-
'functionalSuite1_1',
123-
'functionalSuite1_2',
124-
'functionalSuite1_3'
121+
'functionalSuite1_0_G',
122+
'functionalSuite1_1_G',
123+
'functionalSuite1_2_G',
124+
'functionalSuite1_3_G'
125125
];
126126

127127
$expectedContents = [

‎src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,30 @@
55
*/
66
namespace Magento\FunctionalTestingFramework\Allure\Adapter;
77

8+
use Codeception\Codecept;
9+
use Codeception\Test\Cest;
810
use Codeception\Step\Comment;
911
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
1012
use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject;
1113
use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
1214
use Magento\FunctionalTestingFramework\Util\TestGenerator;
15+
use Yandex\Allure\Adapter\Model\Failure;
16+
use Yandex\Allure\Adapter\Model\Provider;
1317
use Yandex\Allure\Adapter\Model\Status;
1418
use Yandex\Allure\Adapter\Model\Step;
19+
use Yandex\Allure\Adapter\Allure;
1520
use Yandex\Allure\Codeception\AllureCodeception;
1621
use Yandex\Allure\Adapter\Event\StepStartedEvent;
1722
use Yandex\Allure\Adapter\Event\StepFinishedEvent;
1823
use Yandex\Allure\Adapter\Event\StepFailedEvent;
1924
use Yandex\Allure\Adapter\Event\TestCaseFailedEvent;
2025
use Yandex\Allure\Adapter\Event\TestCaseFinishedEvent;
2126
use Yandex\Allure\Adapter\Event\TestCaseBrokenEvent;
27+
use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
2228
use Codeception\Event\FailEvent;
2329
use Codeception\Event\SuiteEvent;
2430
use Codeception\Event\StepEvent;
31+
use Codeception\Event\TestEvent;
2532

2633
/**
2734
* Class MagentoAllureAdapter
@@ -114,6 +121,7 @@ private function sanitizeGroupName($group)
114121
// if we can't find this group in the generated suites we have to assume that the group was split for generation
115122
$groupNameSplit = explode("_", $group);
116123
array_pop($groupNameSplit);
124+
array_pop($groupNameSplit);
117125
$originalName = implode("_", $groupNameSplit);
118126

119127
// confirm our original name is one of the existing suite names otherwise just return the original group name
@@ -245,18 +253,25 @@ public function testError(FailEvent $failEvent)
245253

246254
/**
247255
* Override of parent method, polls stepStorage for testcase and formats it according to actionGroup nesting.
248-
*
256+
* @param TestEvent $testEvent
257+
* @throws \Yandex\Allure\Adapter\AllureException
249258
* @return void
250259
*/
251-
public function testEnd()
260+
public function testEnd(TestEvent $testEvent)
252261
{
262+
$test = $this->getLifecycle()->getTestCaseStorage()->get();
263+
// update testClass label to consolidate re-try reporting
264+
$this->formatAllureTestClassName($test);
265+
// Peek top of testCaseStorage to check of failure
266+
$testFailed = $test->getFailure();
253267
// Pops top of stepStorage, need to add it back in after processing
254268
$rootStep = $this->getLifecycle()->getStepStorage()->pollLast();
255269
$formattedSteps = [];
256270
$actionGroupStepContainer = null;
257271

258272
$actionGroupStepKey = null;
259273
foreach ($rootStep->getSteps() as $step) {
274+
$this->removeAttachments($step, $testFailed);
260275
$stepKey = str_replace($actionGroupStepKey, '', $step->getName());
261276
if ($stepKey !== '[]' && $stepKey !== null) {
262277
$step->setName($stepKey);
@@ -309,9 +324,28 @@ function () use ($rootStep, $formattedSteps) {
309324

310325
$this->getLifecycle()->getStepStorage()->put($rootStep);
311326

327+
$this->addAttachmentEvent($testEvent);
328+
312329
$this->getLifecycle()->fire(new TestCaseFinishedEvent());
313330
}
314331

332+
/**
333+
* Fire add attachment event
334+
* @param TestEvent $testEvent
335+
* @throws \Yandex\Allure\Adapter\AllureException
336+
* @return void
337+
*/
338+
private function addAttachmentEvent(TestEvent $testEvent)
339+
{
340+
// attachments supported since Codeception 3.0
341+
if (version_compare(Codecept::VERSION, '3.0.0') > -1 && $testEvent->getTest() instanceof Cest) {
342+
$artifacts = $testEvent->getTest()->getMetadata()->getReports();
343+
foreach ($artifacts as $name => $artifact) {
344+
Allure::lifecycle()->fire(new AddAttachmentEvent($artifact, $name, null));
345+
}
346+
}
347+
}
348+
315349
/**
316350
* Reads action group stepKey from step.
317351
*
@@ -354,4 +388,60 @@ private function retrieveStepKey($stepLine)
354388

355389
return $stepKey;
356390
}
391+
392+
/**
393+
* Removes attachments from step depending on MFTF configuration
394+
* @param Step $step
395+
* @param Failure $testFailed
396+
* @return void
397+
*/
398+
private function removeAttachments($step, $testFailed)
399+
{
400+
//Remove Attachments if verbose flag is not true AND test did not fail
401+
if (getenv('VERBOSE_ARTIFACTS') !== "true" && $testFailed === null) {
402+
foreach ($step->getAttachments() as $index => $attachment) {
403+
$step->removeAttachment($index);
404+
unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource());
405+
}
406+
}
407+
}
408+
409+
/**
410+
* Format testClass label to consolidate re-try reporting for groups split for parallel execution
411+
* @param TestCase $test
412+
* @return void
413+
*/
414+
private function formatAllureTestClassName($test)
415+
{
416+
if ($this->getGroup() !== null) {
417+
foreach ($test->getLabels() as $name => $label) {
418+
if ($label->getName() == 'testClass') {
419+
$originalTestClass = $this->sanitizeTestClassLabel($label->getValue());
420+
call_user_func(\Closure::bind(
421+
function () use ($label, $originalTestClass) {
422+
$label->value = $originalTestClass;
423+
},
424+
null,
425+
$label
426+
));
427+
break;
428+
}
429+
}
430+
}
431+
}
432+
433+
/**
434+
* Function which sanitizes testClass label for split group runs
435+
* @param string $testClass
436+
* @return string
437+
*/
438+
private function sanitizeTestClassLabel($testClass)
439+
{
440+
$originalTestClass = $testClass;
441+
$originalGroupName = $this->sanitizeGroupName($this->getGroup());
442+
if ($originalGroupName !== $this->getGroup()) {
443+
$originalTestClass = str_replace($this->getGroup(), $originalGroupName, $testClass);
444+
}
445+
return $originalTestClass;
446+
}
357447
}

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,11 +168,7 @@ private function getFailedTestList()
168168
if ($suiteName == self::DEFAULT_TEST_GROUP) {
169169
array_push($failedTestDetails['tests'], $testName);
170170
} else {
171-
// Trim potential suite_parallel_0 to suite_parallel
172-
$suiteNameArray = explode("_", $suiteName);
173-
if (is_numeric(array_pop($suiteNameArray))) {
174-
$suiteName = implode("_", $suiteNameArray);
175-
}
171+
$suiteName = $this->sanitizeSuiteName($suiteName);
176172
$failedTestDetails['suites'] = array_merge_recursive(
177173
$failedTestDetails['suites'],
178174
[$suiteName => [$testName]]
@@ -194,6 +190,23 @@ private function getFailedTestList()
194190
return $testConfigurationJson;
195191
}
196192

193+
/**
194+
* Trim potential suite_parallel_0_G to suite_parallel
195+
*
196+
* @param string $suiteName
197+
* @return string
198+
*/
199+
private function sanitizeSuiteName($suiteName)
200+
{
201+
$suiteNameArray = explode("_", $suiteName);
202+
if (array_pop($suiteNameArray) == 'G') {
203+
if (is_numeric(array_pop($suiteNameArray))) {
204+
$suiteName = implode("_", $suiteNameArray);
205+
}
206+
}
207+
return $suiteName;
208+
}
209+
197210
/**
198211
* Returns an array of run commands read from the manifest file created post generation
199212
*

‎src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -860,8 +860,7 @@ public function saveScreenshot()
860860
*/
861861
public function amOnPage($page)
862862
{
863-
(0 === strpos($page, 'http')) ? parent::amOnUrl($page) : parent::amOnPage($page);
864-
863+
parent::amOnPage($page);
865864
$this->waitForPageLoad();
866865
}
867866

‎src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ private function getSuiteToSize($suiteNamesToTests)
232232
*
233233
* E.g.
234234
* Input {suitename = 'sample', tests = ['test1' => 100,'test2' => 150, 'test3' => 300], linelimit = 275}
235-
* Result { ['sample_01' => ['test3' => 300], 'sample_02' => ['test2' => 150, 'test1' => 100]] }
235+
* Result { ['sample_01_G' => ['test3' => 300], 'sample_02_G' => ['test2' => 150, 'test1' => 100]] }
236236
*
237237
* @param string $suiteName
238238
* @param array $tests
@@ -252,8 +252,8 @@ private function splitTestSuite($suiteName, $tests, $maxTime)
252252
}
253253

254254
$group = $this->createTestGroup($maxTime, $test, $size, $availableTests);
255-
$splitSuites["{$suiteName}_${splitCount}"] = $group;
256-
$this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group);
255+
$splitSuites["{$suiteName}_${splitCount}_G"] = $group;
256+
$this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}_G", $group);
257257

258258
$availableTests = array_diff_key($availableTests, $group);
259259
$splitCount++;

0 commit comments

Comments
 (0)