Skip to content

Commit 958e9c3

Browse files
soumyauKevinBKozan
soumyau
authored and
KevinBKozan
committed
MQE-1541: Add option to generate:tests for XSD validation on 'merged files' (#349)
- added input options to debug argument
1 parent 9359af2 commit 958e9c3

18 files changed

+92
-64
lines changed

‎dev/tests/_bootstrap.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
true,
3333
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
3434
true,
35-
false
35+
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE
3636
);
3737

3838
// Load needed framework env params

‎dev/tests/verification/Resources/BasicFunctionalTest.txt

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,7 @@ class BasicFunctionalTestCest
6060
{
6161
$I->comment("");
6262
$I->comment("");
63-
$I->skipReadinessCheck(true);
64-
$I->comment("skipReadiness");
65-
$I->skipReadinessCheck(false);
63+
$I->comment("seeComment");
6664
$someVarDefinition = $I->grabValueFrom();
6765
$I->acceptPopup();
6866
$I->amOnPage("/test/url");

‎dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,6 @@
115115
<see selector=".selector" userInput="{{sameStepKeyAsArg}}" stepKey="arg1" />
116116
</actionGroup>
117117

118-
<actionGroup name="actionGroupWithSkipReadinessActions">
119-
<comment userInput="ActionGroupSkipReadiness" stepKey="skip" skipReadiness="true"/>
120-
</actionGroup>
121-
122118
<actionGroup name="actionGroupWithSectionAndData">
123119
<arguments>
124120
<argument name="content" type="string"/>

‎dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
</after>
2525
<comment stepKey="basicCommentWithNoData" userInput="{{emptyData.noData}}"/>
2626
<comment stepKey="basicCommentWithDefinitelyNoData" userInput="{{emptyData.definitelyNoData}}"/>
27-
<comment stepKey="ReadinessCheckSkipped" userInput="skipReadiness" skipReadiness="true"/>
27+
<comment stepKey="basicCommentWithData" userInput="seeComment"/>
2828
<grabValueFrom stepKey="someVarDefinition"/>
2929
<acceptPopup stepKey="acceptPopupKey1"/>
3030
<amOnPage stepKey="amOnPageKey1" url="/test/url"/>

‎dev/tests/verification/Tests/ActionGroupGenerationTest.php

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -185,17 +185,6 @@ public function testActionGroupWithArgContainingStepKey()
185185
$this->generateAndCompareTest('ActionGroupContainsStepKeyInArgText');
186186
}
187187

188-
/**
189-
* Test an action group with an arg containing stepKey text
190-
*
191-
* @throws \Exception
192-
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
193-
*/
194-
public function testActionGroupWithSkipReadiness()
195-
{
196-
$this->generateAndCompareTest('ActionGroupSkipReadiness');
197-
}
198-
199188
/**
200189
* Test an action group with an arg containing stepKey text
201190
*

‎dev/tests/verification/Tests/SchemaValidationTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class SchemaValidationTest extends MftfTestCase
1919
*/
2020
public function testInvalidTestSchema()
2121
{
22-
AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]);
22+
AspectMock::double(MftfApplicationConfig::class, ['getDebugLevel' => MftfApplicationConfig::LEVEL_DEVELOPER]);
2323
$testFile = ['testFile.xml' => "<tests><test name='testName'><annotations>a</annotations></test></tests>"];
2424
$expectedError = TESTS_MODULE_PATH .
2525
DIRECTORY_SEPARATOR .

‎src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ class MftfApplicationConfig
1414
const UNIT_TEST_PHASE = "testing";
1515
const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
1616

17+
/**
18+
* Mftf debug levels
19+
*/
20+
const LEVEL_DEFAULT = "default";
21+
const LEVEL_DEVELOPER = "developer";
22+
const LEVEL_NONE = "none";
23+
const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
24+
1725
/**
1826
* Determines whether the user has specified a force option for generation
1927
*
@@ -36,11 +44,11 @@ class MftfApplicationConfig
3644
private $verboseEnabled;
3745

3846
/**
39-
* Determines whether the user would like to execute mftf in a verbose run.
47+
* String which identifies the current debug level of mftf execution
4048
*
41-
* @var boolean
49+
* @var string
4250
*/
43-
private $debugEnabled;
51+
private $debugLevel;
4452

4553
/**
4654
* MftfApplicationConfig Singelton Instance
@@ -55,14 +63,14 @@ class MftfApplicationConfig
5563
* @param boolean $forceGenerate
5664
* @param string $phase
5765
* @param boolean $verboseEnabled
58-
* @param boolean $debugEnabled
66+
* @param string $debugLevel
5967
* @throws TestFrameworkException
6068
*/
6169
private function __construct(
6270
$forceGenerate = false,
6371
$phase = self::EXECUTION_PHASE,
6472
$verboseEnabled = null,
65-
$debugEnabled = null
73+
$debugLevel = self::LEVEL_NONE
6674
) {
6775
$this->forceGenerate = $forceGenerate;
6876

@@ -72,7 +80,15 @@ private function __construct(
7280

7381
$this->phase = $phase;
7482
$this->verboseEnabled = $verboseEnabled;
75-
$this->debugEnabled = $debugEnabled;
83+
switch ($debugLevel) {
84+
case self::LEVEL_DEVELOPER:
85+
case self::LEVEL_DEFAULT:
86+
case self::LEVEL_NONE:
87+
$this->debugLevel = $debugLevel;
88+
break;
89+
default:
90+
$this->debugLevel = self::LEVEL_DEVELOPER;
91+
}
7692
}
7793

7894
/**
@@ -82,14 +98,14 @@ private function __construct(
8298
* @param boolean $forceGenerate
8399
* @param string $phase
84100
* @param boolean $verboseEnabled
85-
* @param boolean $debugEnabled
101+
* @param string $debugLevel
86102
* @return void
87103
*/
88-
public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
104+
public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
89105
{
90106
if (self::$MFTF_APPLICATION_CONTEXT == null) {
91107
self::$MFTF_APPLICATION_CONTEXT =
92-
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
108+
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
93109
}
94110
}
95111

@@ -132,14 +148,13 @@ public function verboseEnabled()
132148
}
133149

134150
/**
135-
* Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
136-
* with some extra error messaging to be run
151+
* Returns a string which indicates the debug level of mftf execution.
137152
*
138-
* @return boolean
153+
* @return string
139154
*/
140-
public function debugEnabled()
155+
public function getDebugLevel()
141156
{
142-
return $this->debugEnabled ?? getenv('MFTF_DEBUG');
157+
return $this->debugLevel ?? getenv('MFTF_DEBUG');
143158
}
144159

145160
/**

‎src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ protected function readFiles($fileList)
157157
} else {
158158
$configMerger->merge($content);
159159
}
160-
if (MftfApplicationConfig::getConfig()->debugEnabled()) {
160+
if (MftfApplicationConfig::getConfig()->getDebugLevel() === MftfApplicationConfig::LEVEL_DEVELOPER) {
161161
$this->validateSchema($configMerger, $fileList->getFilename());
162162
}
163163
} catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
@@ -231,8 +231,14 @@ protected function validateSchema($configMerger, $filename = null)
231231
if ($this->validationState->isValidationRequired()) {
232232
$errors = [];
233233
if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
234-
$message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
235-
throw new \Exception($message . implode("\n", $errors));
234+
foreach ($errors as $error) {
235+
$error = str_replace(PHP_EOL, "", $error);
236+
LoggingUtil::getInstance()->getLogger(Filesystem::class)->criticalFailure(
237+
"Schema validation error ",
238+
($filename ? [ "file"=> $filename, "error" => $error]: ["error" => $error])
239+
);
240+
}
241+
throw new \Exception("Schema validation errors found in xml file(s)" . $filename);
236242
}
237243
}
238244
}

‎src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public function readFiles($fileList)
2424
$exceptionCollector = new ExceptionCollector();
2525
/** @var \Magento\FunctionalTestingFramework\Test\Config\Dom $configMerger */
2626
$configMerger = null;
27+
$debugLevel = MftfApplicationConfig::getConfig()->getDebugLevel();
2728
foreach ($fileList as $key => $content) {
2829
//check if file is empty and continue to next if it is
2930
if (!parent::verifyFileEmpty($content, $fileList->getFilename())) {
@@ -40,16 +41,19 @@ public function readFiles($fileList)
4041
} else {
4142
$configMerger->merge($content, $fileList->getFilename(), $exceptionCollector);
4243
}
43-
if (MftfApplicationConfig::getConfig()->debugEnabled()) {
44+
// run per file validation with generate:tests -d
45+
if ($debugLevel == MftfApplicationConfig::LEVEL_DEVELOPER) {
4446
$this->validateSchema($configMerger, $fileList->getFilename());
4547
}
4648
} catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
4749
throw new \Exception("Invalid XML in file " . $key . ":\n" . $e->getMessage());
4850
}
4951
}
5052
$exceptionCollector->throwException();
51-
if ($fileList->valid()) {
52-
$this->validateSchema($configMerger, $fileList->getFilename());
53+
54+
//run validation on merged file with generate:tests
55+
if ($debugLevel == MftfApplicationConfig::LEVEL_DEFAULT) {
56+
$this->validateSchema($configMerger);
5357
}
5458

5559
$output = [];

‎src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
6767
$force,
6868
MftfApplicationConfig::GENERATION_PHASE,
6969
false,
70-
false
70+
MftfApplicationConfig::LEVEL_NONE
7171
);
7272

7373
$allActionGroups = ActionGroupObjectHandler::getInstance()->getAllObjects();

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class GenerateTestsCommand extends BaseGenerateCommand
2929
protected function configure()
3030
{
3131
$this->setName('generate:tests')
32-
->setDescription('This command generates all test files and suites based on xml declarations')
32+
->setDescription('Run validation and generate all test files and suites based on xml declarations')
3333
->addArgument(
3434
'name',
3535
InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -39,7 +39,7 @@ protected function configure()
3939
"force",
4040
'f',
4141
InputOption::VALUE_NONE,
42-
'force generation of tests regardless of Magento Instance Configuration'
42+
'Force generation of tests regardless of Magento Instance Configuration'
4343
)->addOption(
4444
'time',
4545
'i',
@@ -54,8 +54,10 @@ protected function configure()
5454
)->addOption(
5555
'debug',
5656
'd',
57-
InputOption::VALUE_NONE,
58-
'run extra validation when generating tests'
57+
InputOption::VALUE_OPTIONAL,
58+
'Run extra validation when generating tests. Use option \'none\' to turn off debugging --
59+
added for backward compatibility, will be removed in the next MAJOR release',
60+
'default'
5961
);
6062

6163
parent::configure();
@@ -78,9 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
7880
$json = $input->getOption('tests');
7981
$force = $input->getOption('force');
8082
$time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
81-
$debug = $input->getOption('debug');
83+
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
8284
$remove = $input->getOption('remove');
83-
8485
$verbose = $output->isVerbose();
8586

8687
if ($json !== null && !json_decode($json)) {
@@ -95,7 +96,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
9596

9697
// Remove previous GENERATED_DIR if --remove option is used
9798
if ($remove) {
98-
$this->removeGeneratedDirectory($output, $verbose || $debug);
99+
$this->removeGeneratedDirectory($output, $verbose ||
100+
($debug !== MftfApplicationConfig::LEVEL_NONE));
99101
}
100102

101103
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
@@ -124,13 +126,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
124126
* @param string $json
125127
* @param array $tests
126128
* @param boolean $force
127-
* @param boolean $debug
129+
* @param string $debug
128130
* @param boolean $verbose
129131
* @return array
130132
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
131133
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
132134
*/
133-
private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $verbose)
135+
private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
134136
{
135137
// set our application configuration so we can references the user options in our framework
136138
MftfApplicationConfig::create(

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
namespace Magento\FunctionalTestingFramework\Console;
99

10+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
1011
use Symfony\Component\Console\Input\ArrayInput;
1112
use Symfony\Component\Console\Input\InputArgument;
1213
use Symfony\Component\Console\Input\InputInterface;
@@ -73,7 +74,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7374
'suites' => null
7475
]),
7576
'--force' => $force,
76-
'--remove' => $remove
77+
'--remove' => $remove,
78+
'--debug' => MftfApplicationConfig::LEVEL_NONE
7779
];
7880
$command->run(new ArrayInput($args), $output);
7981
}

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7272
false,
7373
MftfApplicationConfig::GENERATION_PHASE,
7474
false,
75-
false
75+
MftfApplicationConfig::LEVEL_NONE
7676
);
7777

7878
$testConfiguration = $this->getFailedTestList();

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7777
$force,
7878
MftfApplicationConfig::GENERATION_PHASE,
7979
false,
80-
false
80+
MftfApplicationConfig::LEVEL_NONE
8181
);
8282

8383
if (!$skipGeneration) {

‎src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public function execute(InputInterface $input)
7575
true,
7676
MftfApplicationConfig::UNIT_TEST_PHASE,
7777
false,
78-
false
78+
MftfApplicationConfig::LEVEL_NONE
7979
);
8080

8181
ModuleResolver::getInstance()->getModulesPath();

‎src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,6 @@
3131
</xs:annotation>
3232
</xs:attribute>
3333

34-
<xs:attribute type="xs:boolean" name="skipReadiness">
35-
<xs:annotation>
36-
<xs:documentation>
37-
Flag for skipping readiness check
38-
</xs:documentation>
39-
</xs:annotation>
40-
</xs:attribute>
4134
</xs:attributeGroup>
4235

4336
<xs:attribute type="xs:string" name="userInput" id="userInput">

‎src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,13 @@
129129
</xs:documentation>
130130
</xs:annotation>
131131
</xs:attribute>
132+
<xs:attribute type="xs:boolean" name="skipReadiness" use="prohibited">
133+
<xs:annotation>
134+
<xs:documentation>
135+
Flag for skipping readiness check.
136+
</xs:documentation>
137+
</xs:annotation>
138+
</xs:attribute>
132139
<xs:attributeGroup ref="commonActionAttributes"/>
133-
<xs:attribute type="xs:boolean" name="skipReadiness" use="prohibited"/>
134140
</xs:complexType>
135141
</xs:schema>

0 commit comments

Comments
 (0)