Skip to content

Commit 4d91e49

Browse files
jilu1jilu1
authored
Merge branch 'develop' into bugfix/501-documentation
2 parents 46eabbf + afcae2c commit 4d91e49

File tree

8 files changed

+142
-24
lines changed

8 files changed

+142
-24
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,24 @@ public function testThreeTestOneSuiteOneGroupMix()
156156
$this->assertEquals($expected, $actual);
157157
}
158158

159+
public function testSuiteToTestSyntax()
160+
{
161+
$testOne = new TestObject('Test1', [], [], []);
162+
$suiteOne = new SuiteObject(
163+
'Suite1',
164+
['Test1' => $testOne],
165+
[],
166+
[]
167+
);
168+
169+
$testArray = ['Test1' => $testOne];
170+
$suiteArray = ['Suite1' => $suiteOne];
171+
$this->mockHandlers($testArray, $suiteArray);
172+
$actual = json_decode($this->callTestConfig(['Suite1:Test1']), true);
173+
$expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
174+
$this->assertEquals($expected, $actual);
175+
}
176+
159177
/**
160178
* Mock handlers to skip parsing
161179
* @param array $testArray

‎dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Tests\unit\Magento\FunctionalTestFramework\Suite;
77

88
use AspectMock\Test as AspectMock;
9+
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
910
use Magento\FunctionalTestingFramework\ObjectManager\ObjectManager;
1011
use Magento\FunctionalTestingFramework\ObjectManagerFactory;
1112
use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
@@ -17,6 +18,7 @@
1718
use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
1819
use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
1920
use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
21+
use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
2022
use tests\unit\Util\SuiteDataArrayBuilder;
2123
use tests\unit\Util\TestDataArrayBuilder;
2224
use tests\unit\Util\TestLoggingUtil;
@@ -143,6 +145,73 @@ public function testGenerateEmptySuite()
143145
$mockSuiteGenerator->generateSuite("basicTestSuite");
144146
}
145147

148+
public function testInvalidSuiteTestPair()
149+
{
150+
// Mock Suite1 => Test1 and Suite2 => Test2
151+
$suiteDataArrayBuilder = new SuiteDataArrayBuilder();
152+
$mockData = $suiteDataArrayBuilder
153+
->withName('Suite1')
154+
->includeGroups(['group1'])
155+
->build();
156+
$suiteDataArrayBuilder = new SuiteDataArrayBuilder();
157+
$mockData2 = $suiteDataArrayBuilder
158+
->withName('Suite2')
159+
->includeGroups(['group2'])
160+
->build();
161+
$mockSuiteData = array_merge_recursive($mockData, $mockData2);
162+
163+
$testDataArrayBuilder = new TestDataArrayBuilder();
164+
$mockSimpleTest = $testDataArrayBuilder
165+
->withName('Test1')
166+
->withAnnotations(['group' => [['value' => 'group1']]])
167+
->withTestActions()
168+
->build();
169+
$testDataArrayBuilder = new TestDataArrayBuilder();
170+
$mockSimpleTest2 = $testDataArrayBuilder
171+
->withName('Test2')
172+
->withAnnotations(['group' => [['value' => 'group2']]])
173+
->withTestActions()
174+
->build();
175+
$mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
176+
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
177+
178+
// Make invalid manifest
179+
$suiteConfig = ['Suite2' => ['Test1']];
180+
$manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
181+
182+
// Set up Expected Exception
183+
$this->expectException(TestReferenceException::class);
184+
$this->expectExceptionMessageRegExp('(Suite: "Suite2" Tests: "Test1")');
185+
186+
// parse and generate suite object with mocked data and manifest
187+
$mockSuiteGenerator = SuiteGenerator::getInstance();
188+
$mockSuiteGenerator->generateAllSuites($manifest);
189+
}
190+
191+
public function testNonExistentSuiteTestPair()
192+
{
193+
$testDataArrayBuilder = new TestDataArrayBuilder();
194+
$mockSimpleTest = $testDataArrayBuilder
195+
->withName('Test1')
196+
->withAnnotations(['group' => [['value' => 'group1']]])
197+
->withTestActions()
198+
->build();
199+
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
200+
$this->setMockTestAndSuiteParserOutput($mockTestData, []);
201+
202+
// Make invalid manifest
203+
$suiteConfig = ['Suite3' => ['Test1']];
204+
$manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
205+
206+
// Set up Expected Exception
207+
$this->expectException(TestReferenceException::class);
208+
$this->expectExceptionMessageRegExp('#Suite3 is not defined#');
209+
210+
// parse and generate suite object with mocked data and manifest
211+
$mockSuiteGenerator = SuiteGenerator::getInstance();
212+
$mockSuiteGenerator->generateAllSuites($manifest);
213+
}
214+
146215
/**
147216
* Function used to set mock for parser return and force init method to run between tests.
148217
*

‎docs/commands/mftf.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ vendor/bin/mftf generate:tests
4242
vendor/bin/mftf generate:tests AdminLoginTest StorefrontPersistedCustomerLoginTest
4343
```
4444

45+
### Generate test by test and suite name
46+
47+
```bash
48+
vendor/bin/mftf generate:tests LoginSuite:AdminLoginTest
49+
```
50+
4551
### Generate and run the tests for a specified group
4652

4753
```bash
@@ -58,6 +64,14 @@ vendor/bin/mftf run:test AdminLoginTest StorefrontPersistedCustomerLoginTest -r
5864

5965
This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests.
6066

67+
### Generate and run particular test in a specific suite's context
68+
69+
```bash
70+
vendor/bin/mftf run:test LoginSuite:AdminLoginTest -r
71+
```
72+
73+
This command cleans up previously generated tests; generates and run `AdminLoginTest` within the context of the `LoginSuite`.
74+
6175
### Generate and run a testManifest.txt file
6276

6377
```bash

‎docs/extending.md

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ __Use case__: Create two similar tests with different `url` (`"{{AdminCategoryPa
2626
> Test with "extends":
2727
2828
```xml
29-
<tests >
29+
<tests>
3030
<test name="AdminCategoryTest">
3131
<annotations>
3232
...
@@ -47,7 +47,7 @@ __Use case__: Create two similar tests with different `url` (`"{{AdminCategoryPa
4747
> Test without "extends":
4848
4949
```xml
50-
<tests >
50+
<tests>
5151
<test name="AdminCategoryTest">
5252
<annotations>
5353
...
@@ -77,7 +77,7 @@ __Use case__: Create two similar tests where the second test contains two additi
7777
> Tests with "extends":
7878
7979
```xml
80-
<tests >
80+
<tests>
8181
<test name="LogInAsAdminTest">
8282
<amOnPage url="{{AdminLoginPage}}" stepKey="navigateToAdmin"/>
8383
<fillField selector="{{AdminLoginFormSection.username}}" userInput="admin" stepKey="fillUsername"/>
@@ -95,7 +95,7 @@ __Use case__: Create two similar tests where the second test contains two additi
9595
> Tests without "extends":
9696
9797
```xml
98-
<tests >
98+
<tests>
9999
<test name="LogInAsAdminTest">
100100
<amOnPage url="{{AdminLoginPage}}" stepKey="navigateToAdmin"/>
101101
<fillField selector="{{AdminLoginFormSection.username}}" userInput="admin" stepKey="fillUsername"/>
@@ -125,7 +125,7 @@ __Use case__: Create two similar tests where the second one contains two additio
125125
> Tests with "extends":
126126
127127
```xml
128-
<tests >
128+
<tests>
129129
<test name="LogInAsAdminTest">
130130
<before>
131131
<amOnPage url="{{AdminLoginPage}}" stepKey="navigateToAdmin"/>
@@ -147,7 +147,7 @@ __Use case__: Create two similar tests where the second one contains two additio
147147
> Tests without "extends":
148148
149149
```xml
150-
<tests >
150+
<tests>
151151
<test name="LogInAsAdminTest">
152152
<before>
153153
<amOnPage url="{{AdminLoginPage}}" stepKey="navigateToAdmin"/>
@@ -295,7 +295,7 @@ __Use case__: Create an entity named `DivPanelGreen`, which is similar to the `D
295295
> Entities with "extends":
296296
297297
```xml
298-
<entities >
298+
<entities>
299299
<entity name="DivPanel">
300300
<data key="divColor">Red</data>
301301
<data key="divSize">80px</data>
@@ -310,7 +310,7 @@ __Use case__: Create an entity named `DivPanelGreen`, which is similar to the `D
310310
> Entities without "extends":
311311
312312
```xml
313-
<entities >
313+
<entities>
314314
<entity name="DivPanel">
315315
<data key="divColor">Red</data>
316316
<data key="divSize">80px</data>
@@ -331,7 +331,7 @@ __Use case__: Create an entity named `DivPanelGreen`, which is similar to the `D
331331
> Entities with "extends":
332332
333333
```xml
334-
<entities >
334+
<entities>
335335
<entity name="DivPanel">
336336
<data key="divColor">Red</data>
337337
<data key="divSize">80px</data>
@@ -347,7 +347,7 @@ __Use case__: Create an entity named `DivPanelGreen`, which is similar to the `D
347347
> Entities without "extends":
348348
349349
```xml
350-
<entities >
350+
<entities>
351351
<entity name="DivPanel">
352352
<data key="divColor">Red</data>
353353
<data key="divSize">80px</data>

‎src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ protected function getTestAndSuiteConfiguration(array $tests)
8787
$suiteToTestPair = [];
8888

8989
foreach($tests as $test) {
90+
if (strpos($test, ':') !== false) {
91+
$suiteToTestPair[] = $test;
92+
continue;
93+
}
9094
if (array_key_exists($test, $testsReferencedInSuites)) {
9195
$suites = $testsReferencedInSuites[$test];
9296
foreach ($suites as $suite) {

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,21 +94,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9494
$command->run(new ArrayInput($args), $output);
9595
}
9696

97-
$codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps';
97+
$commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps';
9898

99+
$exitCode = -1;
100+
$returnCodes = [];
99101
foreach ($groups as $group) {
100-
$codeceptionCommand .= " -g {$group}";
101-
}
102+
$codeceptionCommandString = $commandString . " -g {$group}";
103+
104+
$process = new Process($codeceptionCommandString);
105+
$process->setWorkingDirectory(TESTS_BP);
106+
$process->setIdleTimeout(600);
107+
$process->setTimeout(0);
102108

103-
$process = new Process($codeceptionCommand);
104-
$process->setWorkingDirectory(TESTS_BP);
105-
$process->setIdleTimeout(600);
106-
$process->setTimeout(0);
109+
$returnCodes[] = $process->run(
110+
function ($type, $buffer) use ($output) {
111+
$output->write($buffer);
112+
}
113+
);
114+
}
107115

108-
return $process->run(
109-
function ($type, $buffer) use ($output) {
110-
$output->write($buffer);
116+
foreach ($returnCodes as $returnCode) {
117+
if ($returnCode != 0) {
118+
return $returnCode;
111119
}
112-
);
120+
$exitCode = 0;
121+
}
122+
return $exitCode;
113123
}
114124
}

‎src/Magento/FunctionalTestingFramework/Suite/Handlers/SuiteObjectHandler.php

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66
namespace Magento\FunctionalTestingFramework\Suite\Handlers;
77

8+
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
89
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
910
use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface;
1011
use Magento\FunctionalTestingFramework\ObjectManagerFactory;
@@ -73,7 +74,7 @@ public static function getInstance(): ObjectHandlerInterface
7374
public function getObject($objectName): SuiteObject
7475
{
7576
if (!array_key_exists($objectName, $this->suiteObjects)) {
76-
trigger_error("Suite ${objectName} is not defined.", E_USER_ERROR);
77+
throw new TestReferenceException("Suite ${objectName} is not defined in xml.");
7778
}
7879
return $this->suiteObjects[$objectName];
7980
}

‎src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,12 +178,14 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o
178178
{
179179
$suiteRef = $originalSuiteName ?? $suiteName;
180180
$possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
181-
$errorMsg = "Cannot reference tests whcih are not declared as part of suite.";
181+
$errorMsg = "Cannot reference tests which are not declared as part of suite";
182182

183183
$invalidTestRef = array_diff($testsReferenced, array_keys($possibleTestRef));
184184

185185
if (!empty($invalidTestRef)) {
186-
throw new TestReferenceException($errorMsg, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
186+
$testList = implode("\", \"", $invalidTestRef);
187+
$fullError = $errorMsg . " (Suite: \"{$suiteRef}\" Tests: \"{$testList}\")";
188+
throw new TestReferenceException($fullError, ['suite' => $suiteRef, 'test' => $invalidTestRef]);
187189
}
188190
}
189191

0 commit comments

Comments
 (0)