Skip to content

Commit 6977262

Browse files
jilu1jilu1
committed
MQE-2114: Add additional checks to the annotations static check
1 parent 7b6ec51 commit 6977262

File tree

4 files changed

+272
-9
lines changed

4 files changed

+272
-9
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/StaticCheck/AnnotationsCheckTest.php

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace tests\unit\Magento\FunctionalTestFramework\StaticCheck;
8+
9+
use AspectMock\Test as AspectMock;
10+
use Magento\FunctionalTestingFramework\StaticCheck\AnnotationsCheck;
11+
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
12+
use tests\unit\Util\MagentoTestCase;
13+
use ReflectionClass;
14+
15+
class AnnotationsCheckTest extends MagentoTestCase
16+
{
17+
/** @var AnnotationsCheck */
18+
private $staticCheck;
19+
20+
/** @var ReflectionClass*/
21+
private $staticCheckClass;
22+
23+
public function setUp(): void
24+
{
25+
$this->staticCheck = new AnnotationsCheck();
26+
$this->staticCheckClass = new \ReflectionClass($this->staticCheck);
27+
}
28+
29+
public function tearDown(): void
30+
{
31+
AspectMock::clean();
32+
}
33+
34+
public function testValidateRequiredAnnotationsNoError()
35+
{
36+
$annotations = [
37+
'features' => [
38+
0 => 'feature1'
39+
],
40+
'stories' => [
41+
0 => 'story1'
42+
],
43+
'description' => [
44+
'main' => 'description1',
45+
'test_files' => 'file1',
46+
'deprecated' => [
47+
0 => 'deprecated1'
48+
]
49+
],
50+
'severity' => [
51+
0 => 'severity1'
52+
],
53+
'title' => [
54+
0 => '[NO TESTCASEID]: title1'
55+
],
56+
];
57+
$expected = [];
58+
59+
// mock test object
60+
$test = AspectMock::double(
61+
TestObject::class, ['getAnnotations' => $annotations, 'getName' => 'AnnotationsCheckTest']
62+
)->make();
63+
64+
$validateRequiredAnnotations = $this->staticCheckClass->getMethod('validateRequiredAnnotations');
65+
$validateRequiredAnnotations->setAccessible(true);
66+
67+
$validateRequiredAnnotations->invoke($this->staticCheck, $test);
68+
$this->assertEquals($expected, $this->staticCheck->getErrors());
69+
}
70+
71+
public function testValidateRequiredAnnotationsMissing()
72+
{
73+
$testCaseId = 'MC-12345';
74+
75+
$annotations = [
76+
'features' => [
77+
0 => 'feature1'
78+
],
79+
'stories' => [
80+
0 => 'story1'
81+
],
82+
'description' => [
83+
'test_files' => 'file1',
84+
'deprecated' => [
85+
0 => 'deprecated1'
86+
]
87+
],
88+
'title' => [
89+
0 => $testCaseId . ': title1'
90+
],
91+
'testCaseId' => [
92+
0 => $testCaseId
93+
]
94+
];
95+
$expected = [
96+
0 => [
97+
0 => 'Test AnnotationsCheckTest is missing the required annotations: description, severity'
98+
]
99+
];
100+
101+
// mock test object
102+
$test = AspectMock::double(
103+
TestObject::class, ['getAnnotations' => $annotations, 'getName' => 'AnnotationsCheckTest']
104+
)->make();
105+
106+
$validateRequiredAnnotations = $this->staticCheckClass->getMethod('validateRequiredAnnotations');
107+
$validateRequiredAnnotations->setAccessible(true);
108+
109+
$validateRequiredAnnotations->invoke($this->staticCheck, $test);
110+
$this->assertEquals($expected, $this->staticCheck->getErrors());
111+
}
112+
113+
public function testValidateRequiredAnnotationsMissingNoTestCaseId()
114+
{
115+
$annotations = [
116+
'features' => [
117+
0 => 'feature1'
118+
],
119+
'stories' => [
120+
0 => 'story1'
121+
],
122+
'description' => [
123+
'test_files' => 'file1',
124+
'deprecated' => [
125+
0 => 'deprecated1'
126+
]
127+
],
128+
'title' => [
129+
0 => "[NO TESTCASEID]: \t"
130+
],
131+
];
132+
$expected = [
133+
0 => [
134+
0 => 'Test AnnotationsCheckTest is missing the required annotations: title, description, severity'
135+
]
136+
];
137+
138+
// mock test object
139+
$test = AspectMock::double(
140+
TestObject::class, ['getAnnotations' => $annotations, 'getName' => 'AnnotationsCheckTest']
141+
)->make();
142+
143+
$validateRequiredAnnotations = $this->staticCheckClass->getMethod('validateRequiredAnnotations');
144+
$validateRequiredAnnotations->setAccessible(true);
145+
146+
$validateRequiredAnnotations->invoke($this->staticCheck, $test);
147+
$this->assertEquals($expected, $this->staticCheck->getErrors());
148+
}
149+
150+
public function testValidateRequiredAnnotationsEmpty()
151+
{
152+
$annotations = [
153+
'features' => [
154+
0 => 'feature1'
155+
],
156+
'stories' => [
157+
0 => 'story1'
158+
],
159+
'description' => [
160+
'main' => 'description1',
161+
'test_files' => 'file1',
162+
'deprecated' => [
163+
0 => 'deprecated1'
164+
]
165+
],
166+
'severity' => [
167+
0 => 'severity1'
168+
],
169+
'title' => [
170+
0 => ''
171+
],
172+
];
173+
$expected = [
174+
0 => [
175+
0 => 'Test AnnotationsCheckTest is missing the required annotations: title'
176+
]
177+
];
178+
179+
// mock test object
180+
$test = AspectMock::double(
181+
TestObject::class, ['getAnnotations' => $annotations, 'getName' => 'AnnotationsCheckTest']
182+
)->make();
183+
184+
$validateRequiredAnnotations = $this->staticCheckClass->getMethod('validateRequiredAnnotations');
185+
$validateRequiredAnnotations->setAccessible(true);
186+
187+
$validateRequiredAnnotations->invoke($this->staticCheck, $test);
188+
$this->assertEquals($expected, $this->staticCheck->getErrors());
189+
}
190+
}

‎dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/AnnotationExtractorTest.php

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,68 @@ public function testMissingAnnotations()
122122
);
123123
}
124124

125+
/**
126+
* Annotation extractor should throw warning when required annotations are empty
127+
*
128+
* @throws \Exception
129+
*/
130+
public function testEmptyRequiredAnnotations()
131+
{
132+
// Test Data, missing title, description, and severity
133+
$testAnnotations = [
134+
"nodeName" => "annotations",
135+
"features" => [
136+
[
137+
"nodeName" => "features",
138+
"value" => ""
139+
]
140+
],
141+
"stories" => [
142+
[
143+
"nodeName" => "stories",
144+
"value" => "TestStories"
145+
]
146+
],
147+
"title" => [
148+
[
149+
"nodeName" => "title",
150+
"value" => " "
151+
]
152+
],
153+
"description" => [
154+
[
155+
"nodeName" => "description",
156+
"value" => "\t"
157+
]
158+
],
159+
"severity" => [
160+
[
161+
"nodeName" => "severity",
162+
"value" => ""
163+
]
164+
],
165+
"group" => [
166+
[
167+
"nodeName" => "group",
168+
"value" => "TestGroup"
169+
]
170+
],
171+
];
172+
// Perform Test
173+
$extractor = new AnnotationExtractor();
174+
$returnedAnnotations = $extractor->extractAnnotations($testAnnotations, "testFileName");
175+
176+
// Asserts
177+
TestLoggingUtil::getInstance()->validateMockLogStatement(
178+
'warning',
179+
'DEPRECATION: Test testFileName is missing required annotations.',
180+
[
181+
'testName' => 'testFileName',
182+
'missingAnnotations' => "title, description, severity"
183+
]
184+
);
185+
}
186+
125187
public function testTestCaseIdUniqueness()
126188
{
127189
// Test Data

‎src/Magento/FunctionalTestingFramework/StaticCheck/AnnotationsCheck.php

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,22 +124,30 @@ private function validateRequiredAnnotations($test)
124124
$missing = [];
125125

126126
$stories = $annotations['stories'] ?? null;
127-
if ($stories === null) {
127+
if ($stories === null || !isset($stories[0]) || empty(trim($stories[0]))) {
128128
$missing[] = "stories";
129129
}
130130

131+
$testCaseId = "[NO TESTCASEID]";
132+
if (isset($annotations['testCaseId'][0])) {
133+
$testCaseId = trim($annotations['testCaseId'][0]);
134+
}
135+
131136
$title = $annotations['title'] ?? null;
132-
if ($title === null) {
137+
if ($title === null
138+
|| !isset($title[0])
139+
|| empty(trim($title[0]))
140+
|| empty(trim(substr(trim($title[0]), strlen($testCaseId . ': '))))) {
133141
$missing[] = "title";
134142
}
135143

136144
$description = $annotations['description']['main'] ?? null;
137-
if ($description === null) {
145+
if ($description === null || empty(trim($description))) {
138146
$missing[] = "description";
139147
}
140148

141149
$severity = $annotations['severity'] ?? null;
142-
if ($severity === null) {
150+
if ($severity === null || !isset($severity[0]) || empty(trim($severity[0]))) {
143151
$missing[] = "severity";
144152
}
145153

‎src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public function extractAnnotations($testAnnotations, $filename, $validateAnnotat
7777
// Only transform severity annotation
7878
if ($annotationKey == "severity") {
7979
$annotationObjects[$annotationKey] = $this->transformAllureSeverityToMagento(
80-
$annotationData[0]['value']
80+
trim($annotationData[0][self::ANNOTATION_VALUE])
8181
);
8282
continue;
8383
}
@@ -90,16 +90,17 @@ public function extractAnnotations($testAnnotations, $filename, $validateAnnotat
9090
}
9191

9292
foreach ($annotationData as $annotationValue) {
93-
$annotationValues[] = $annotationValue[self::ANNOTATION_VALUE];
93+
$annotationValues[] = trim($annotationValue[self::ANNOTATION_VALUE]);
9494
}
9595

9696
$annotationObjects[$annotationKey] = $annotationValues;
9797
}
9898

99-
$this->addTestCaseIdToTitle($annotationObjects, $filename);
10099
if ($validateAnnotations) {
101100
$this->validateMissingAnnotations($annotationObjects, $filename);
102101
}
102+
103+
$this->addTestCaseIdToTitle($annotationObjects, $filename);
103104
$this->addStoryTitleToMap($annotationObjects, $filename);
104105

105106
return $annotationObjects;
@@ -154,7 +155,9 @@ private function validateMissingAnnotations($annotationObjects, $filename)
154155
$missingAnnotations = [];
155156

156157
foreach (self::REQUIRED_ANNOTATIONS as $REQUIRED_ANNOTATION) {
157-
if (!array_key_exists($REQUIRED_ANNOTATION, $annotationObjects)) {
158+
if (!array_key_exists($REQUIRED_ANNOTATION, $annotationObjects)
159+
|| !isset($annotationObjects[$REQUIRED_ANNOTATION][0])
160+
|| empty($annotationObjects[$REQUIRED_ANNOTATION][0])) {
158161
$missingAnnotations[] = $REQUIRED_ANNOTATION;
159162
}
160163
}
@@ -231,7 +234,7 @@ public function validateTestCaseIdTitleUniqueness()
231234
public function validateSkippedIssues($issues, $filename)
232235
{
233236
foreach ($issues as $issueId) {
234-
if (empty($issueId['value'])) {
237+
if (empty(trim($issueId['value']))) {
235238
$message = "issueId for skipped tests cannot be empty. Test: $filename";
236239
throw new XmlException($message);
237240
}

0 commit comments

Comments
 (0)