Skip to content

Commit 241734e

Browse files
imeron2433imeron2433
authored
Merge pull request #118 from magento/MQE-982
MQE-982: Verification Test around Suite Generation
2 parents a955ad1 + 163b913 commit 241734e

File tree

6 files changed

+356
-19
lines changed

6 files changed

+356
-19
lines changed

‎dev/tests/_bootstrap.php

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,27 @@
6262
require($unitUtilFile);
6363
}
6464

65+
66+
// Mocks suite files location getter return to get files in verification/_suite Directory
67+
// This mocks the paths of the suite files but still parses the xml files
68+
$suiteDirectory = TESTS_BP . DIRECTORY_SEPARATOR . "verification" . DIRECTORY_SEPARATOR . "_suite";
69+
70+
$paths = [
71+
$suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuite.xml',
72+
$suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuiteHooks.xml'
73+
];
74+
75+
// create and return the iterator for these file paths
76+
$iterator = new Magento\FunctionalTestingFramework\Util\Iterator\File($paths);
77+
try {
78+
AspectMock\Test::double(
79+
Magento\FunctionalTestingFramework\Config\FileResolver\Root::class,
80+
['get' => $iterator]
81+
)->make();
82+
} catch (Exception $e) {
83+
echo "Suite directory not mocked.";
84+
}
85+
6586
function sortInterfaces($files)
6687
{
6788
$bottom = [];

‎dev/tests/verification/Resources/functionalSuiteHooks.txt

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
<?php
2+
3+
namespace Group;
4+
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
8+
/**
9+
* Group class is Codeception Extension which is allowed to handle to all internal events.
10+
* This class itself can be used to listen events for test execution of one particular group.
11+
* It may be especially useful to create fixtures data, prepare server, etc.
12+
*
13+
* INSTALLATION:
14+
*
15+
* To use this group extension, include it to "extensions" option of global Codeception config.
16+
*/
17+
class functionalSuiteHooks extends \Codeception\GroupObject
18+
{
19+
public static $group = 'functionalSuiteHooks';
20+
private $testCount = 1;
21+
private $preconditionFailure = null;
22+
private $currentTestRun = 0;
23+
private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of functionalSuiteHooks suite %s block ********/\n";
24+
private static $HOOK_EXECUTION_END = "\n/******** Execution of functionalSuiteHooks suite %s block complete ********/\n";
25+
private $create;
26+
27+
public function _before(\Codeception\Event\TestEvent $e)
28+
{
29+
// increment test count per execution
30+
$this->currentTestRun++;
31+
$this->executePreConditions();
32+
33+
if ($this->preconditionFailure != null) {
34+
//if our preconditions fail, we need to mark all the tests as incomplete.
35+
$e->getTest()->getMetadata()->setIncomplete($this->preconditionFailure);
36+
}
37+
}
38+
39+
40+
private function executePreConditions()
41+
{
42+
if ($this->currentTestRun == 1) {
43+
print sprintf(self::$HOOK_EXECUTION_INIT, "before");
44+
45+
try {
46+
$webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver');
47+
48+
// close any open sessions
49+
if ($webDriver->webDriver != null) {
50+
$webDriver->webDriver->close();
51+
$webDriver->webDriver = null;
52+
}
53+
54+
// initialize the webdriver session
55+
$webDriver->_initializeSession();
56+
$webDriver->amOnPage("some.url");
57+
$createFields['someKey'] = "dataHere";
58+
$createThis = DataObjectHandler::getInstance()->getObject("createThis");
59+
$this->create = new DataPersistenceHandler($createThis, [], $createFields);
60+
$this->create->createEntity();
61+
62+
// reset configuration and close session
63+
$this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig();
64+
$webDriver->webDriver->close();
65+
$webDriver->webDriver = null;
66+
} catch (\Exception $exception) {
67+
$this->preconditionFailure = $exception->getMessage();
68+
}
69+
70+
print sprintf(self::$HOOK_EXECUTION_END, "before");
71+
}
72+
}
73+
74+
public function _after(\Codeception\Event\TestEvent $e)
75+
{
76+
$this->executePostConditions();
77+
}
78+
79+
80+
private function executePostConditions()
81+
{
82+
if ($this->currentTestRun == $this->testCount) {
83+
print sprintf(self::$HOOK_EXECUTION_INIT, "after");
84+
85+
try {
86+
$webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver');
87+
88+
// close any open sessions
89+
if ($webDriver->webDriver != null) {
90+
$webDriver->webDriver->close();
91+
$webDriver->webDriver = null;
92+
}
93+
94+
// initialize the webdriver session
95+
$webDriver->_initializeSession();
96+
$webDriver->amOnPage("some.url");
97+
$webDriver->deleteEntityByUrl("deleteThis");
98+
99+
// reset configuration and close session
100+
$this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig();
101+
$webDriver->webDriver->close();
102+
$webDriver->webDriver = null;
103+
} catch (\Exception $exception) {
104+
print $exception->getMessage();
105+
}
106+
107+
print sprintf(self::$HOOK_EXECUTION_END, "after");
108+
}
109+
}
110+
}

‎dev/tests/verification/Tests/SuiteGenerationTest.php

Lines changed: 181 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,23 @@
66

77
namespace tests\verification\Tests;
88

9-
use Magento\Framework\Module\Dir;
109
use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
11-
use Magento\FunctionalTestingFramework\Util\TestManifest;
10+
use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
11+
use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest;
12+
use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
1213
use Symfony\Component\Yaml\Yaml;
1314
use tests\util\MftfTestCase;
1415

1516
class SuiteGenerationTest extends MftfTestCase
1617
{
1718
const RESOURCES_DIR = TESTS_BP . DIRECTORY_SEPARATOR . 'verification' . DIRECTORY_SEPARATOR . 'Resources';
1819
const CONFIG_YML_FILE = FW_BP . DIRECTORY_SEPARATOR . SuiteGenerator::YAML_CODECEPTION_CONFIG_FILENAME;
20+
const GENERATE_RESULT_DIR = TESTS_BP .
21+
DIRECTORY_SEPARATOR .
22+
"verification" .
23+
DIRECTORY_SEPARATOR .
24+
"_generated" .
25+
DIRECTORY_SEPARATOR;
1926

2027
/**
2128
* Flag to track existence of config.yml file
@@ -41,9 +48,9 @@ public static function setUpBeforeClass()
4148
return;
4249
}
4350

44-
// destroy manifest file if it exists
45-
if (file_exists(self::getManifestFilePath())) {
46-
unlink(self::getManifestFilePath());
51+
// destroy _generated if it exists
52+
if (file_exists(self::GENERATE_RESULT_DIR)) {
53+
DirSetupUtil::rmdirRecursive(self::GENERATE_RESULT_DIR);
4754
}
4855

4956
$configYml = fopen(self::CONFIG_YML_FILE, "w");
@@ -55,7 +62,7 @@ public static function setUpBeforeClass()
5562
*/
5663
public function testSuiteGeneration1()
5764
{
58-
$groupName = 'functionalSuite1';
65+
$groupName = 'functionalSuite1';
5966

6067
$expectedContents = [
6168
'additionalTestCest.php',
@@ -75,11 +82,159 @@ public function testSuiteGeneration1()
7582
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
7683
$this->assertArrayHasKey($groupName, $yml['groups']);
7784

78-
$suiteResultBaseDir = TESTS_BP .
85+
$suiteResultBaseDir = self::GENERATE_RESULT_DIR .
7986
DIRECTORY_SEPARATOR .
80-
"verification" .
87+
$groupName .
88+
DIRECTORY_SEPARATOR;
89+
90+
// Validate tests have been generated
91+
$dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
92+
93+
foreach ($expectedContents as $expectedFile) {
94+
$this->assertTrue(in_array($expectedFile, $dirContents));
95+
}
96+
}
97+
98+
/**
99+
* Test generation of parallel suite groups
100+
*/
101+
public function testSuiteGenerationParallel()
102+
{
103+
$groupName = 'functionalSuite1';
104+
105+
$expectedGroups = [
106+
'functionalSuite1_0',
107+
'functionalSuite1_1',
108+
'functionalSuite1_2',
109+
'functionalSuite1_3'
110+
];
111+
112+
$expectedContents = [
113+
'additionalTestCest.php',
114+
'additionalIncludeTest2Cest.php',
115+
'IncludeTest2Cest.php',
116+
'IncludeTestCest.php'
117+
];
118+
119+
//createParallelManifest
120+
/** @var ParallelTestManifest $parallelManifest */
121+
$parallelManifest = TestManifestFactory::makeManifest("parallel", ["functionalSuite1" => []]);
122+
123+
// Generate the Suite
124+
$parallelManifest->createTestGroups(1);
125+
SuiteGenerator::getInstance()->generateAllSuites($parallelManifest);
126+
127+
// Validate console message and add group name for later deletion
128+
$this->expectOutputRegex('/Suite .* generated to .*/');
129+
self::$TEST_GROUPS[] = $groupName;
130+
131+
// Validate Yaml file updated
132+
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
133+
$this->assertEquals(array_intersect($expectedGroups, array_keys($yml['groups'])), $expectedGroups);
134+
135+
foreach ($expectedGroups as $expectedFolder) {
136+
$suiteResultBaseDir = self::GENERATE_RESULT_DIR .
137+
DIRECTORY_SEPARATOR .
138+
$expectedFolder .
139+
DIRECTORY_SEPARATOR;
140+
141+
// Validate tests have been generated
142+
$dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
143+
144+
//Validate only one test has been added to each group since lines are set to 1
145+
$this->assertEquals(1, count($dirContents));
146+
$this->assertContains(array_values($dirContents)[0], $expectedContents);
147+
}
148+
}
149+
150+
/**
151+
* Test hook groups generated during suite generation
152+
*/
153+
public function testSuiteGenerationHooks()
154+
{
155+
$groupName = 'functionalSuiteHooks';
156+
157+
$expectedContents = [
158+
'IncludeTestCest.php'
159+
];
160+
161+
// Generate the Suite
162+
SuiteGenerator::getInstance()->generateSuite($groupName);
163+
164+
// Validate console message and add group name for later deletion
165+
$this->expectOutputRegex('/Suite .* generated to .*/');
166+
self::$TEST_GROUPS[] = $groupName;
167+
168+
// Validate Yaml file updated
169+
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
170+
$this->assertArrayHasKey($groupName, $yml['groups']);
171+
172+
$suiteResultBaseDir = self::GENERATE_RESULT_DIR .
81173
DIRECTORY_SEPARATOR .
82-
"_generated" .
174+
$groupName .
175+
DIRECTORY_SEPARATOR;
176+
177+
// Validate tests have been generated
178+
$dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
179+
180+
foreach ($expectedContents as $expectedFile) {
181+
$this->assertTrue(in_array($expectedFile, $dirContents));
182+
}
183+
184+
//assert group file created and contains correct contents
185+
$groupFile = PROJECT_ROOT .
186+
DIRECTORY_SEPARATOR .
187+
"src" .
188+
DIRECTORY_SEPARATOR .
189+
"Magento" .
190+
DIRECTORY_SEPARATOR .
191+
"FunctionalTestingFramework" .
192+
DIRECTORY_SEPARATOR .
193+
"Group" .
194+
DIRECTORY_SEPARATOR .
195+
$groupName .
196+
".php";
197+
198+
$this->assertTrue(file_exists($groupFile));
199+
$this->assertFileEquals(
200+
self::RESOURCES_PATH . DIRECTORY_SEPARATOR . $groupName . ".txt",
201+
$groupFile
202+
);
203+
204+
}
205+
206+
/**
207+
* Test generation of parallel suite groups
208+
*/
209+
public function testSuiteGenerationSingleRun()
210+
{
211+
//using functionalSuite2 to avoid directory caching
212+
$groupName = 'functionalSuite2';
213+
214+
$expectedContents = [
215+
'additionalTestCest.php',
216+
'additionalIncludeTest2Cest.php',
217+
'IncludeTest2Cest.php',
218+
'IncludeTestCest.php'
219+
];
220+
221+
//createParallelManifest
222+
/** @var ParallelTestManifest $parallelManifest */
223+
$singleRunManifest = TestManifestFactory::makeManifest("singleRun", ["functionalSuite2" => []]);
224+
225+
// Generate the Suite
226+
SuiteGenerator::getInstance()->generateAllSuites($singleRunManifest);
227+
$singleRunManifest->generate();
228+
229+
// Validate console message and add group name for later deletion
230+
$this->expectOutputRegex('/Suite .* generated to .*/');
231+
self::$TEST_GROUPS[] = $groupName;
232+
233+
// Validate Yaml file updated
234+
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
235+
$this->assertArrayHasKey($groupName, $yml['groups']);
236+
237+
$suiteResultBaseDir = self::GENERATE_RESULT_DIR .
83238
DIRECTORY_SEPARATOR .
84239
$groupName .
85240
DIRECTORY_SEPARATOR;
@@ -90,25 +245,38 @@ public function testSuiteGeneration1()
90245
foreach ($expectedContents as $expectedFile) {
91246
$this->assertTrue(in_array($expectedFile, $dirContents));
92247
}
248+
249+
$expectedManifest = "dev/tests/verification/_generated/default/" . PHP_EOL . "-g functionalSuite2" . PHP_EOL;
250+
251+
$this->assertEquals($expectedManifest, file_get_contents(self::getManifestFilePath()));
93252
}
94253

95254
/**
96255
* revert any changes made to config.yml
256+
* remove _generated directory
97257
*/
98-
public static function tearDownAfterClass()
258+
public function tearDown()
99259
{
100260
// restore config if we see there was an original codeception.yml file
101261
if (self::$YML_EXISTS_FLAG) {
102262
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
103263
foreach (self::$TEST_GROUPS as $testGroup) {
104-
unset($yml['group'][$testGroup]);
264+
unset($yml['groups'][$testGroup]);
105265
}
106266

107267
file_put_contents(self::CONFIG_YML_FILE, Yaml::dump($yml, 10));
108-
return;
109268
}
269+
DirSetupUtil::rmdirRecursive(self::GENERATE_RESULT_DIR);
270+
}
110271

111-
unlink(self::CONFIG_YML_FILE);
272+
/**
273+
* Remove yml if created during tests and did not exist before
274+
*/
275+
public static function tearDownAfterClass()
276+
{
277+
if (!self::$YML_EXISTS_FLAG) {
278+
unlink(self::CONFIG_YML_FILE);
279+
}
112280
}
113281

114282
/**

0 commit comments

Comments
 (0)