Skip to content

Commit 0bef784

Browse files
jilu1jilu1
committed
MQE-2494: Missing steps in Allure logs for certain test failures
1 parent f80e2db commit 0bef784

File tree

8 files changed

+68
-100
lines changed

8 files changed

+68
-100
lines changed

‎dev/tests/verification/Resources/AssertTest.txt

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ class AssertTestCest
4343
$I->comment("asserts without variable replacement");
4444
$I->assertArrayHasKey("apple", ['orange' => 2, 'apple' => 1], "pass"); // stepKey: assertArrayHasKey
4545
$I->assertArrayNotHasKey("kiwi", ['orange' => 2, 'apple' => 1], "pass"); // stepKey: assertArrayNotHasKey
46-
$I->assertArraySubset([1, 2], [1, 2, 3, 5], "pass"); // stepKey: assertArraySubset
4746
$I->assertContains("ab", ['item1' => 'a', 'item2' => 'ab'], "pass"); // stepKey: assertContains
4847
$I->assertStringContainsString("ab", "abcde", "pass"); // stepKey: assertStringContainsString
48+
$I->assertStringContainsString("ab", $var, "pass"); // stepKey: assertStringContainsStringVar
49+
$I->assertContains("ab", $var, "pass"); // stepKey: assertContainsVar
4950
$I->assertCount(2, ['a', 'b'], "pass"); // stepKey: assertCount
5051
$I->assertEmpty([], "pass"); // stepKey: assertEmpty
5152
$I->assertEquals($text, "Copyright © 2013-2017 Magento, Inc. All rights reserved.", "pass"); // stepKey: assertEquals1
@@ -64,7 +65,9 @@ class AssertTestCest
6465
$I->assertLessThan(5, 2, "pass"); // stepKey: assertLessThan
6566
$I->assertLessThanOrEqual(5, 2, "pass"); // stepKey: assertLessThanOrEquals
6667
$I->assertNotContains("bc", ['item1' => 'a', 'item2' => 'ab'], "pass"); // stepKey: assertNotContains
67-
$I->assertStringNotContainsString("bc", $text, "pass"); // stepKey: assertStringNotContainsString
68+
$I->assertStringNotContainsString("bc", "text", "pass"); // stepKey: assertStringNotContainsString
69+
$I->assertNotContains("bc", $arrayVar, "pass"); // stepKey: assertNotContainsVar
70+
$I->assertStringNotContainsString("bc", $var, "pass"); // stepKey: assertStringNotContainsStringVar
6871
$I->assertNotEmpty([1, 2], "pass"); // stepKey: assertNotEmpty1
6972
$I->assertNotEmpty($text, "pass"); // stepKey: assertNotEmpty2
7073
$I->assertNotEqualsWithDelta(2, 5, 0, "pass"); // stepKey: assertNotEquals
@@ -81,9 +84,10 @@ class AssertTestCest
8184
$I->comment("asserts backward compatible");
8285
$I->assertArrayHasKey("apple", ['orange' => 2, 'apple' => 1], "pass"); // stepKey: assertArrayHasKeyBackwardCompatible
8386
$I->assertArrayNotHasKey("kiwi", ['orange' => 2, 'apple' => 1], "pass"); // stepKey: assertArrayNotHasKeyBackwardCompatible
84-
$I->assertArraySubset([1, 2], [1, 2, 3, 5], "pass"); // stepKey: assertArraySubsetBackwardCompatible
8587
$I->assertContains("ab", ['item1' => 'a', 'item2' => 'ab'], "pass"); // stepKey: assertContains1BackwardCompatible
8688
$I->assertStringContainsString("ab", "abcde", "pass"); // stepKey: assertContains2BackwardCompatible
89+
$I->assertStringContainsString("ab", $var, "pass"); // stepKey: assertContains3BackwardCompatible
90+
$I->assertContains("ab", $arrayVar, "pass"); // stepKey: assertContains4BackwardCompatible
8791
$I->assertCount(2, ['a', 'b'], "pass"); // stepKey: assertCountBackwardCompatible
8892
$I->assertEmpty([], "pass"); // stepKey: assertEmptyBackwardCompatible
8993
$I->assertEquals($text, "Copyright © 2013-2017 Magento, Inc. All rights reserved.", "pass"); // stepKey: assertEquals1BackwardCompatible
@@ -101,7 +105,9 @@ class AssertTestCest
101105
$I->assertLessThan(5, 2, "pass"); // stepKey: assertLessThanBackwardCompatible
102106
$I->assertLessThanOrEqual(5, 2, "pass"); // stepKey: assertLessThanOrEqualBackwardCompatible
103107
$I->assertNotContains("bc", ['item1' => 'a', 'item2' => 'ab'], "pass"); // stepKey: assertNotContains1BackwardCompatible
104-
$I->assertStringNotContainsString("bc", $text, "pass"); // stepKey: assertNotContains2BackwardCompatible
108+
$I->assertStringNotContainsString("bc", "text", "pass"); // stepKey: assertNotContains2BackwardCompatible
109+
$I->assertNotContains("bc", $arrayVar, "pass"); // stepKey: assertNotContains3BackwardCompatible
110+
$I->assertStringNotContainsString("bc", $var, "pass"); // stepKey: assertNotContains4BackwardCompatible
105111
$I->assertNotEmpty([1, 2], "pass"); // stepKey: assertNotEmpty1BackwardCompatible
106112
$I->assertNotEmpty($text, "pass"); // stepKey: assertNotEmpty2BackwardCompatible
107113
$I->assertNotEqualsWithDelta(2, 5, 0, "pass"); // stepKey: assertNotEqualsBackwardCompatible
@@ -128,8 +134,6 @@ class AssertTestCest
128134
$I->assertEquals($I->retrieveEntityField('createData1', 'lastname', 'test'), $I->retrieveEntityField('createData1', 'lastname', 'test'), "pass"); // stepKey: assert5
129135
$I->comment("array type that use created data");
130136
$I->comment("array type that use created data");
131-
$I->assertArraySubset([$I->retrieveEntityField('createData1', 'lastname', 'test'), $I->retrieveEntityField('createData1', 'firstname', 'test')], [$I->retrieveEntityField('createData1', 'lastname', 'test'), $I->retrieveEntityField('createData1', 'firstname', 'test'), "1"], "pass"); // stepKey: assert9
132-
$I->assertArraySubset([$I->retrieveEntityField('createData2', 'firstname', 'test'), $I->retrieveEntityField('createData2', 'lastname', 'test')], [$I->retrieveEntityField('createData2', 'firstname', 'test'), $I->retrieveEntityField('createData2', 'lastname', 'test'), "1"], "pass"); // stepKey: assert10
133137
$I->assertArrayHasKey("lastname", ['lastname' => $I->retrieveEntityField('createData1', 'lastname', 'test'), 'firstname' => $I->retrieveEntityField('createData1', 'firstname', 'test')], "pass"); // stepKey: assert3
134138
$I->assertArrayHasKey("lastname", ['lastname' => $I->retrieveEntityField('createData2', 'lastname', 'test'), 'firstname' => $I->retrieveEntityField('createData2', 'firstname', 'test')], "pass"); // stepKey: assert4
135139
$I->comment("this section can only be generated and cannot run");

‎dev/tests/verification/Resources/BasicFunctionalTest.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ class BasicFunctionalTestCest
138138
$I->moveForward(); // stepKey: moveForwardKey1
139139
$I->moveMouseOver(".functionalTestSelector"); // stepKey: moveMouseOverKey1
140140
$I->openNewTab(); // stepKey: openNewTabKey1
141-
$I->pauseExecution(); // stepKey: pauseExecutionKey1
141+
$I->pause(); // stepKey: pauseExecutionKey1
142142
$I->performOn("#selector", function(\WebDriverElement $el) {return $el->isDisplayed();}, 10); // stepKey: performOnKey1
143143
$I->pressKey("#page", "a"); // stepKey: pressKey1
144144
$I->pressKey("#page", ['ctrl', 'a'],'new'); // stepKey: pressKey2

‎dev/tests/verification/TestModule/Test/AssertTest.xml

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,6 @@
3131
<expectedResult type="string">kiwi</expectedResult>
3232
<actualResult type="const">['orange' => 2, 'apple' => 1]</actualResult>
3333
</assertArrayNotHasKey>
34-
<assertArraySubset stepKey="assertArraySubset" message="pass">
35-
<expectedResult type="const">[1, 2]</expectedResult>
36-
<actualResult type="const">[1, 2, 3, 5]</actualResult>
37-
</assertArraySubset>
3834
<assertContains stepKey="assertContains" message="pass">
3935
<expectedResult type="string">ab</expectedResult>
4036
<actualResult type="const">['item1' => 'a', 'item2' => 'ab']</actualResult>
@@ -43,6 +39,14 @@
4339
<expectedResult type="string">ab</expectedResult>
4440
<actualResult type="string">abcde</actualResult>
4541
</assertContains>
42+
<assertContains stepKey="assertStringContainsStringVar" message="pass">
43+
<expectedResult type="string">ab</expectedResult>
44+
<actualResult type="variable">var</actualResult>
45+
</assertContains>
46+
<assertContains stepKey="assertContainsVar" message="pass">
47+
<expectedResult type="string">ab</expectedResult>
48+
<actualResult type="arrayVariable">var</actualResult>
49+
</assertContains>
4650
<assertCount stepKey="assertCount" message="pass">
4751
<expectedResult type="int">2</expectedResult>
4852
<actualResult type="const">['a', 'b']</actualResult>
@@ -113,7 +117,15 @@
113117
</assertNotContains>
114118
<assertNotContains stepKey="assertStringNotContainsString" message="pass">
115119
<expectedResult type="string">bc</expectedResult>
116-
<actualResult type="variable">text</actualResult>
120+
<actualResult type="string">text</actualResult>
121+
</assertNotContains>
122+
<assertNotContains stepKey="assertNotContainsVar" message="pass">
123+
<expectedResult type="string">bc</expectedResult>
124+
<actualResult type="arrayVariable">arrayVar</actualResult>
125+
</assertNotContains>
126+
<assertNotContains stepKey="assertStringNotContainsStringVar" message="pass">
127+
<expectedResult type="string">bc</expectedResult>
128+
<actualResult type="variable">var</actualResult>
117129
</assertNotContains>
118130
<assertNotEmpty stepKey="assertNotEmpty1" message="pass">
119131
<actualResult type="const">[1, 2]</actualResult>
@@ -163,9 +175,10 @@
163175
<comment stepKey="commentBackwardCompatible" userInput="asserts backward compatible"/>
164176
<assertArrayHasKey stepKey="assertArrayHasKeyBackwardCompatible" expected="apple" expectedType="string" actual="['orange' => 2, 'apple' => 1]" actualType="const" message="pass"/>
165177
<assertArrayNotHasKey stepKey="assertArrayNotHasKeyBackwardCompatible" expected="kiwi" expectedType="string" actual="['orange' => 2, 'apple' => 1]" message="pass"/>
166-
<assertArraySubset stepKey="assertArraySubsetBackwardCompatible" expected="[1, 2]" actual="[1, 2, 3, 5]" message="pass"/>
167178
<assertContains stepKey="assertContains1BackwardCompatible" expected="ab" expectedType="string" actual="['item1' => 'a', 'item2' => 'ab']" message="pass"/>
168179
<assertContains stepKey="assertContains2BackwardCompatible" expected="ab" expectedType="string" actual="abcde" actualType="string" message="pass"/>
180+
<assertContains stepKey="assertContains3BackwardCompatible" expected="ab" expectedType="string" actual="var" actualType="variable" message="pass"/>
181+
<assertContains stepKey="assertContains4BackwardCompatible" expected="ab" expectedType="string" actual="arrayVar" actualType="arrayVariable" message="pass"/>
169182
<assertCount stepKey="assertCountBackwardCompatible" expected="2" expectedType="int" actual="['a', 'b']" message="pass"/>
170183
<assertEmpty stepKey="assertEmptyBackwardCompatible" actual="[]" message="pass"/>
171184
<assertEquals stepKey="assertEquals1BackwardCompatible" expected="text" expectedType="variable" actual="Copyright © 2013-2017 Magento, Inc. All rights reserved." actualType="string" message="pass"/>
@@ -183,7 +196,9 @@
183196
<assertLessThan stepKey="assertLessThanBackwardCompatible" expected="5" expectedType="int" actual="2" actualType="int" message="pass"/>
184197
<assertLessThanOrEqual stepKey="assertLessThanOrEqualBackwardCompatible" expected="5" expectedType="int" actual="2" actualType="int" message="pass"/>
185198
<assertNotContains stepKey="assertNotContains1BackwardCompatible" expected="bc" expectedType="string" actual="['item1' => 'a', 'item2' => 'ab']" message="pass"/>
186-
<assertNotContains stepKey="assertNotContains2BackwardCompatible" expected="bc" expectedType="string" actual="text" actualType="variable" message="pass"/>
199+
<assertNotContains stepKey="assertNotContains2BackwardCompatible" expected="bc" expectedType="string" actual="text" actualType="string" message="pass"/>
200+
<assertNotContains stepKey="assertNotContains3BackwardCompatible" expected="bc" expectedType="string" actual="arrayVar" actualType="arrayVariable" message="pass"/>
201+
<assertNotContains stepKey="assertNotContains4BackwardCompatible" expected="bc" expectedType="string" actual="var" actualType="variable" message="pass"/>
187202
<assertNotEmpty stepKey="assertNotEmpty1BackwardCompatible" actual="[1, 2]" message="pass"/>
188203
<assertNotEmpty stepKey="assertNotEmpty2BackwardCompatible" actual="text" actualType="variable" message="pass"/>
189204
<assertNotEquals stepKey="assertNotEqualsBackwardCompatible" expected="2" expectedType="int" actual="5" actualType="int" message="pass" delta=""/>
@@ -221,14 +236,6 @@
221236

222237
<!-- array type that use created data -->
223238
<comment stepKey="c3" userInput="array type that use created data"/>
224-
<assertArraySubset stepKey="assert9" message="pass">
225-
<expectedResult type="array">[$$createData1.lastname$$, $$createData1.firstname$$]</expectedResult>
226-
<actualResult type="array">[$$createData1.lastname$$, $$createData1.firstname$$, 1]</actualResult>
227-
</assertArraySubset>
228-
<assertArraySubset stepKey="assert10" message="pass">
229-
<expectedResult type="array">[$createData2.firstname$, $createData2.lastname$]</expectedResult>
230-
<actualResult type="array">[$createData2.firstname$, $createData2.lastname$, 1]</actualResult>
231-
</assertArraySubset>
232239
<assertArrayHasKey stepKey="assert3" message="pass">
233240
<expectedResult type="string">lastname</expectedResult>
234241
<actualResult type="array">['lastname' => $$createData1.lastname$$, 'firstname' => $$createData1.firstname$$]</actualResult>

‎docs/test/assertions.md

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -117,22 +117,6 @@ Attribute|Type|Use|Description
117117
`before`|string|optional| `stepKey` of action that must be executed next.
118118
`after`|string|optional| `stepKey` of the preceding action.
119119

120-
### assertArraySubset
121-
122-
See [assertArraySubset docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertArraySubset).
123-
124-
Attribute|Type|Use|Description
125-
---|---|---|---
126-
`expected`|string|required| A value of the expected result.
127-
`expectedType`|string|optional| A type of the expected result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
128-
`actual`|string|required| A value of the actual result.
129-
`actualType`|string|optional| A type of the actual result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
130-
`strict`|boolean|optional|
131-
`message`|string|optional|Text of informational message about a cause of failure.
132-
`stepKey`|string|required| A unique identifier of the text step.
133-
`before`|string|optional| `stepKey` of action that must be executed next.
134-
`after`|string|optional| `stepKey` of the preceding action.
135-
136120
### assertContains
137121

138122
See [assertContains docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertContains).
@@ -142,7 +126,7 @@ Attribute|Type|Use|Description
142126
`expected`|string|required| A value of the expected result.
143127
`expectedType`|string|optional| A type of the expected result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
144128
`actual`|string|required| A value of the actual result.
145-
`actualType`|string|optional| A type of the actual result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
129+
`actualType`|string|optional| A type of the actual result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`, `arrayVariable`.
146130
`message`|string|optional|Text of informational message about a cause of failure.
147131
`stepKey`|string|required| A unique identifier of the text step.
148132
`before`|string|optional| `stepKey` of action that must be executed next.
@@ -373,7 +357,7 @@ Attribute|Type|Use|Description
373357
`expected`|string|required| A value of the expected result.
374358
`expectedType`|string|optional| A type of the expected result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
375359
`actual`|string|required| A value of the actual result.
376-
`actualType`|string|optional| A type of the actual result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`.
360+
`actualType`|string|optional| A type of the actual result. Possible values: `const` (default), `int`, `float`, `bool`, `string`, `variable`, `array`, `arrayVariable`.
377361
`message`|string|optional|Text of informational message about a cause of failure.
378362
`stepKey`|string|required| A unique identifier of the text step.
379363
`before`|string|optional| `stepKey` of action that must be executed next.

0 commit comments

Comments
 (0)