Skip to content

Commit 485c49b

Browse files
jilu1jilu1
committed
Merge remote-tracking branch 'origin/develop' into MQE-1257
2 parents bea39a9 + 6c13d9d commit 485c49b

File tree

6 files changed

+377
-44
lines changed

6 files changed

+377
-44
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace Tests\unit\Magento\FunctionalTestingFramework\Console;
7+
8+
use AspectMock\Test as AspectMock;
9+
use PHPUnit\Framework\TestCase;
10+
use Magento\FunctionalTestingFramework\Console\BaseGenerateCommand;
11+
use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
12+
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
13+
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
14+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
15+
16+
class BaseGenerateCommandTest extends TestCase
17+
{
18+
public function tearDown()
19+
{
20+
AspectMock::clean();
21+
}
22+
23+
public function testOneTestOneSuiteConfig()
24+
{
25+
$testOne = new TestObject('Test1', [], [], []);
26+
$suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
27+
28+
$testArray = ['Test1' => $testOne];
29+
$suiteArray = ['Suite1' => $suiteOne];
30+
31+
$this->mockHandlers($testArray, $suiteArray);
32+
33+
$actual = json_decode($this->callTestConfig(['Test1']), true);
34+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
35+
$this->assertEquals($expected, $actual);
36+
}
37+
38+
public function testOneTestTwoSuitesConfig()
39+
{
40+
$testOne = new TestObject('Test1', [], [], []);
41+
$suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
42+
$suiteTwo = new SuiteObject('Suite2', ['Test1' => $testOne], [], []);
43+
44+
$testArray = ['Test1' => $testOne];
45+
$suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo];
46+
47+
$this->mockHandlers($testArray, $suiteArray);
48+
49+
$actual = json_decode($this->callTestConfig(['Test1']), true);
50+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test1']]];
51+
$this->assertEquals($expected, $actual);
52+
}
53+
54+
public function testOneTestOneGroup()
55+
{
56+
$testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
57+
58+
$testArray = ['Test1' => $testOne];
59+
$suiteArray = [];
60+
61+
$this->mockHandlers($testArray, $suiteArray);
62+
63+
$actual = json_decode($this->callGroupConfig(['Group1']), true);
64+
$expected = ['tests' => ['Test1'], 'suites' => null];
65+
$this->assertEquals($expected, $actual);
66+
}
67+
68+
public function testThreeTestsTwoGroup()
69+
{
70+
$testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
71+
$testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []);
72+
$testThree = new TestObject('Test3', [], ['group' => ['Group2']], []);
73+
74+
$testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree];
75+
$suiteArray = [];
76+
77+
$this->mockHandlers($testArray, $suiteArray);
78+
79+
$actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true);
80+
$expected = ['tests' => ['Test1', 'Test2', 'Test3'], 'suites' => null];
81+
$this->assertEquals($expected, $actual);
82+
}
83+
84+
public function testOneTestOneSuiteOneGroupConfig()
85+
{
86+
$testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
87+
$suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
88+
89+
$testArray = ['Test1' => $testOne];
90+
$suiteArray = ['Suite1' => $suiteOne];
91+
92+
$this->mockHandlers($testArray, $suiteArray);
93+
94+
$actual = json_decode($this->callGroupConfig(['Group1']), true);
95+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
96+
$this->assertEquals($expected, $actual);
97+
}
98+
99+
public function testTwoTestOneSuiteTwoGroupConfig()
100+
{
101+
$testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
102+
$testTwo = new TestObject('Test2', [], ['group' => ['Group2']], []);
103+
$suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne, 'Test2' => $testTwo], [], []);
104+
105+
$testArray = ['Test1' => $testOne, 'Test2' => $testTwo];
106+
$suiteArray = ['Suite1' => $suiteOne];
107+
108+
$this->mockHandlers($testArray, $suiteArray);
109+
110+
$actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true);
111+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1', 'Test2']]];
112+
$this->assertEquals($expected, $actual);
113+
}
114+
115+
public function testTwoTestTwoSuiteOneGroupConfig()
116+
{
117+
$testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
118+
$testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []);
119+
$suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
120+
$suiteTwo = new SuiteObject('Suite2', ['Test2' => $testTwo], [], []);
121+
122+
$testArray = ['Test1' => $testOne, 'Test2' => $testTwo];
123+
$suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo];
124+
125+
$this->mockHandlers($testArray, $suiteArray);
126+
127+
$actual = json_decode($this->callGroupConfig(['Group1']), true);
128+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test2']]];
129+
$this->assertEquals($expected, $actual);
130+
}
131+
132+
/**
133+
* Test specific usecase of a test that is in a group with the group being called along with the suite
134+
* i.e. run:group Group1 Suite1
135+
* @throws \Exception
136+
*/
137+
public function testThreeTestOneSuiteOneGroupMix()
138+
{
139+
$testOne = new TestObject('Test1', [], [], []);
140+
$testTwo = new TestObject('Test2', [], [], []);
141+
$testThree = new TestObject('Test3', [], ['group' => ['Group1']], []);
142+
$suiteOne = new SuiteObject(
143+
'Suite1',
144+
['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree],
145+
[],
146+
[]
147+
);
148+
149+
$testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree];
150+
$suiteArray = ['Suite1' => $suiteOne];
151+
152+
$this->mockHandlers($testArray, $suiteArray);
153+
154+
$actual = json_decode($this->callGroupConfig(['Group1', 'Suite1']), true);
155+
$expected = ['tests' => null, 'suites' => ['Suite1' => []]];
156+
$this->assertEquals($expected, $actual);
157+
}
158+
159+
/**
160+
* Mock handlers to skip parsing
161+
* @param array $testArray
162+
* @param array $suiteArray
163+
* @throws \Exception
164+
*/
165+
public function mockHandlers($testArray, $suiteArray)
166+
{
167+
AspectMock::double(TestObjectHandler::class, ['initTestData' => ''])->make();
168+
$handler = TestObjectHandler::getInstance();
169+
$property = new \ReflectionProperty(TestObjectHandler::class, 'tests');
170+
$property->setAccessible(true);
171+
$property->setValue($handler, $testArray);
172+
173+
AspectMock::double(SuiteObjectHandler::class, ['initSuiteData' => ''])->make();
174+
$handler = SuiteObjectHandler::getInstance();
175+
$property = new \ReflectionProperty(SuiteObjectHandler::class, 'suiteObjects');
176+
$property->setAccessible(true);
177+
$property->setValue($handler, $suiteArray);
178+
}
179+
180+
/**
181+
* Changes visibility and runs getTestAndSuiteConfiguration
182+
* @param array $testArray
183+
* @return string
184+
*/
185+
public function callTestConfig($testArray)
186+
{
187+
$command = new BaseGenerateCommand();
188+
$class = new \ReflectionClass($command);
189+
$method = $class->getMethod('getTestAndSuiteConfiguration');
190+
$method->setAccessible(true);
191+
return $method->invokeArgs($command, [$testArray]);
192+
}
193+
194+
/**
195+
* Changes visibility and runs getGroupAndSuiteConfiguration
196+
* @param array $groupArray
197+
* @return string
198+
*/
199+
public function callGroupConfig($groupArray)
200+
{
201+
$command = new BaseGenerateCommand();
202+
$class = new \ReflectionClass($command);
203+
$method = $class->getMethod('getGroupAndSuiteConfiguration');
204+
$method->setAccessible(true);
205+
return $method->invokeArgs($command, [$groupArray]);
206+
}
207+
}

‎docs/credentials.md

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Credentials
22

3-
When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD,
3+
When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD,
44
use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys.
55

66
Currently the MFTF supports two types of credential storage:
@@ -53,9 +53,9 @@ magento/carriers_usps_password=Lmgxvrq89uPwECeV
5353
#magento/carriers_dhl_id_us=dhl_test_user
5454
#magento/carriers_dhl_password_us=Mlgxv3dsagVeG
5555
....
56-
```
56+
```
5757

58-
Or add new key & value pairs for your own credentials. The keys use the following format:
58+
Or add new key/value pairs for your own credentials. The keys use the following format:
5959

6060
```conf
6161
<vendor>/<key_name>=<key_value>
@@ -64,7 +64,7 @@ Or add new key & value pairs for your own credentials. The keys use the followin
6464
<div class="bs-callout bs-callout-info" markdown="1">
6565
The `/` symbol is not supported in a `key_name` other than the one after your vendor or extension name.
6666
</div>
67-
67+
6868
Otherwise you are free to use any other `key_name` you like, as they are merely the keys to reference from your tests.
6969

7070
```conf
@@ -74,10 +74,10 @@ vendor/my_awesome_service_token=rRVSVnh3cbDsVG39oTMz4A
7474

7575
## Configure Vault Storage
7676

77-
Hashicorp vault secures, stores, and tightly controls access to data in modern computing.
78-
It provides advanced data protection for your testing credentials.
77+
Hashicorp vault secures, stores, and tightly controls access to data in modern computing.
78+
It provides advanced data protection for your testing credentials.
7979

80-
The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine.
80+
The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine.
8181

8282
### Install vault CLI
8383

@@ -95,8 +95,8 @@ vault login -method -path
9595

9696
### Store secrets in vault
9797

98-
The MFTF uses the `KV Version 2` secret engine for secret storage.
99-
More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2].
98+
The MFTF uses the `KV Version 2` secret engine for secret storage.
99+
More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2].
100100

101101
#### Secrets path and key convention
102102

@@ -125,9 +125,9 @@ vault kv put secret/mftf/magento/carriers_usps_password carriers_usps_password=L
125125

126126
### Setup MFTF to use vault
127127

128-
Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][]
128+
Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][]
129129
from `etc/config/.env.example` in `.env`.
130-
Set values according to your vault server configuration.
130+
Set values according to your vault server configuration.
131131

132132
```conf
133133
# Default vault dev server
@@ -137,7 +137,7 @@ CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
137137

138138
## Configure both File Storage and Vault Storage
139139

140-
It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time.
140+
It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time.
141141
In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence.
142142

143143
<!-- {% raw %} -->
@@ -150,11 +150,12 @@ Define the value as a reference to the corresponding key in the credentials file
150150

151151
- `_CREDS` is an environment constant pointing to the `.credentials` file
152152
- `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step
153+
- for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key`
153154

154-
For example, reference secret data in the [`fillField`][] action with the `userInput` attribute.
155+
For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage:
155156

156157
```xml
157-
<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.my_data_key}}" />
158+
<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.vendor/my_data_key}}" />
158159
```
159160

160161
<!-- {% endraw %} -->

‎docs/guides/action-groups.md

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Action Group Best Practices
2+
3+
We strive to write tests using only action groups. Fortunately, we have built up a large set of action groups to get started. We can make use of them and extend them for our own specific needs. In some cases, we may never even need to write action groups of our own. We may be able to simply chain together calls to existing action groups to implement our new test case.
4+
5+
## Why use Action Groups?
6+
7+
Action groups simplify maintainability by reducing duplication. Because they are re-usable building blocks, odds are that they are already made use of by existing tests in the Magento codebase. This proves their stability through real-world use. Take for example, the action group named `LoginAsAdmin`:
8+
9+
```xml
10+
<actionGroup name="LoginAsAdmin">
11+
<annotations>
12+
<description>Login to Backend Admin using provided User Data. PLEASE NOTE: This Action Group does NOT validate that you are Logged In.</description>
13+
</annotations>
14+
<arguments>
15+
<argument name="adminUser" type="entity" defaultValue="DefaultAdminUser"/>
16+
</arguments>
17+
18+
<amOnPage url="{{AdminLoginPage.url}}" stepKey="navigateToAdmin"/>
19+
<fillField selector="{{AdminLoginFormSection.username}}" userInput="{{adminUser.username}}" stepKey="fillUsername"/>
20+
<fillField selector="{{AdminLoginFormSection.password}}" userInput="{{adminUser.password}}" stepKey="fillPassword"/>
21+
<click selector="{{AdminLoginFormSection.signIn}}" stepKey="clickLogin"/>
22+
<closeAdminNotification stepKey="closeAdminNotification"/>
23+
</actionGroup>
24+
```
25+
26+
Logging in to the admin panel is one of the most used action groups. It is used around 1,500 times at the time of this writing.
27+
28+
Imagine if this was not an action group and instead we were to copy and paste these 5 actions every time. In that scenario, if a small change was needed, it would require a lot of work. But with the action group, we can make the change in one place.
29+
30+
## How to extend action groups
31+
32+
Again using `LoginAsAdmin` as our example, we trim away metadata to clearly reveal that this action group performs 5 actions:
33+
34+
```xml
35+
<actionGroup name="LoginAsAdmin">
36+
...
37+
<amOnPage url="{{AdminLoginPage.url}}" .../>
38+
<fillField selector="{{AdminLoginFormSection.username}}" .../>
39+
<fillField selector="{{AdminLoginFormSection.password}}" .../>
40+
<click selector="{{AdminLoginFormSection.signIn}}" .../>
41+
<closeAdminNotification .../>
42+
</actionGroup>
43+
```
44+
45+
This works against the standard Magento admin panel login page. Bu imagine we are working on a Magento extension that adds a CAPTCHA field to the login page. If we create and activate this extension and then run all existing tests, we can expect almost everything to fail because the CAPTCHA field is left unfilled.
46+
47+
We can overcome this by making use of MFTF's extensibility. All we need to do is to provide a "merge" that modifies the existing `LoginAsAdmin` action group. Our merge file will look like:
48+
49+
```xml
50+
<actionGroup name="LoginAsAdmin">
51+
<fillField selector="{{CaptchaSection.captchaInput}}" before="signIn" .../>
52+
</actionGroup>
53+
```
54+
55+
Because the name of this merge is also `LoginAsAdmin`, the two get merged together and an additional step happens everytime this action group is used.
56+
57+
To continue this example, imagine someone else is working on a 'Two-Factor Authentication' extension and they also provide a merge for the `LoginAsAdmin` action group. Their merge looks similar to what we have already seen. The only difference is that this time we fill a different field:
58+
59+
```xml
60+
<actionGroup name="LoginAsAdmin">
61+
<fillField selector="{{TwoFactorSection.twoFactorInput}}" before="signIn" .../>
62+
</actionGroup>
63+
```
64+
65+
Bringing it all together, our resulting `LoginAsAdmin` action group becomes this:
66+
67+
```xml
68+
<actionGroup name="LoginAsAdmin">
69+
...
70+
<amOnPage url="{{AdminLoginPage.url}}" .../>
71+
<fillField selector="{{AdminLoginFormSection.username}}" .../>
72+
<fillField selector="{{AdminLoginFormSection.password}}" .../>
73+
<fillField selector="{{CaptchaSection.captchaInput}}" .../>
74+
<fillField selector="{{TwoFactorSection.twoFactorInput}}" .../>
75+
<click selector="{{AdminLoginFormSection.signIn}}" .../>
76+
<closeAdminNotification .../>
77+
</actionGroup>
78+
```
79+
80+
No one file contains this exact content as above, but instead all three files come together to form this action group.
81+
82+
This extensibility can be applied in many ways. We can use it to affect existing Magento entities such as tests, action groups, and data. Not so obvious is that this tehcnique can be used within your own entities to make them more maintainable as well.

‎etc/config/command.php

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@
3131
$process->run();
3232
$output = $process->getOutput();
3333
if (!$process->isSuccessful()) {
34-
$output = $process->getErrorOutput();
34+
$failureOutput = $process->getErrorOutput();
35+
if (!empty($failureOutput)) {
36+
$output = $failureOutput;
37+
}
3538
}
3639
if (empty($output)) {
3740
$output = "CLI did not return output.";

0 commit comments

Comments
 (0)