Skip to content

Commit fecaf5b

Browse files
KevinBKozanKevinBKozan
authored
Merge pull request #429 from magento/MQE-1582
MQE-1582: Enable Testers To Run Skipped Tests
2 parents 4e259a0 + c37280f commit fecaf5b

File tree

11 files changed

+135
-65
lines changed

11 files changed

+135
-65
lines changed

‎dev/tests/_bootstrap.php

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@
3232
true,
3333
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
3434
true,
35-
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE
35+
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE,
36+
false
3637
);
3738

3839
// Load needed framework env params

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
1313
use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
1414
use Magento\FunctionalTestingFramework\Util\TestGenerator;
15+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
1516

1617
class TestGeneratorTest extends MagentoTestCase
1718
{
@@ -38,4 +39,51 @@ public function testEntityException()
3839

3940
$testGeneratorObject->createAllTestFiles(null, []);
4041
}
42+
43+
/**
44+
* Tests that skipped tests do not have a fully generated body
45+
*
46+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
47+
*/
48+
public function testSkippedNoGeneration()
49+
{
50+
$actionInput = 'fakeInput';
51+
$actionObject = new ActionObject('fakeAction', 'comment', [
52+
'userInput' => $actionInput
53+
]);
54+
55+
$annotations = ['skip' => ['issue']];
56+
$testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename");
57+
58+
$testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]);
59+
$output = $testGeneratorObject->assembleTestPhp($testObject);
60+
61+
$this->assertContains('This test is skipped', $output);
62+
$this->assertNotContains($actionInput, $output);
63+
}
64+
65+
/**
66+
* Tests that skipped tests have a fully generated body when --allowSkipped is passed in
67+
*
68+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
69+
*/
70+
public function testAllowSkipped()
71+
{
72+
// Mock allowSkipped for TestGenerator
73+
AspectMock::double(MftfApplicationConfig::class, ['allowSkipped' => true]);
74+
75+
$actionInput = 'fakeInput';
76+
$actionObject = new ActionObject('fakeAction', 'comment', [
77+
'userInput' => $actionInput
78+
]);
79+
80+
$annotations = ['skip' => ['issue']];
81+
$testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename");
82+
83+
$testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]);
84+
$output = $testGeneratorObject->assembleTestPhp($testObject);
85+
86+
$this->assertNotContains('This test is skipped', $output);
87+
$this->assertContains($actionInput, $output);
88+
}
4189
}

‎docs/commands/mftf.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
122122
| `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
123123
| `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
124124
| `--tests` | Defines the test configuration as a JSON string.|
125+
| `--allow-skipped` | Allows MFTF to generate and run tests marked with `<skip>.`|
125126
| `--debug or --debug=[<none>]`| Performs schema validations on XML files. <br/> DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. <br/> DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.</br><br/> NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.</br>|
126127
| `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.|
127128

‎etc/config/.env.example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_C
4949
#*** Bool property which allows the user to toggle debug output during test execution
5050
#MFTF_DEBUG=
5151

52+
#*** Bool property which allows the user to generate and run tests marked as skipped
53+
#ALLOW_SKIPPED=true
54+
5255
#*** Default timeout for wait actions
5356
#WAIT_TIMEOUT=10
5457
#*** End of .env ***#

‎src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99

1010
class MftfApplicationConfig
1111
{
12+
/**
13+
* MFTF Execution Phases
14+
*/
1215
const GENERATION_PHASE = "generation";
1316
const EXECUTION_PHASE = "execution";
1417
const UNIT_TEST_PHASE = "testing";
@@ -50,6 +53,12 @@ class MftfApplicationConfig
5053
*/
5154
private $debugLevel;
5255

56+
/**
57+
* Boolean which allows MFTF to fully generate skipped tests
58+
* @var boolean
59+
*/
60+
private $allowSkipped;
61+
5362
/**
5463
* MftfApplicationConfig Singelton Instance
5564
*
@@ -64,13 +73,15 @@ class MftfApplicationConfig
6473
* @param string $phase
6574
* @param boolean $verboseEnabled
6675
* @param string $debugLevel
76+
* @param boolean $allowSkipped
6777
* @throws TestFrameworkException
6878
*/
6979
private function __construct(
7080
$forceGenerate = false,
7181
$phase = self::EXECUTION_PHASE,
7282
$verboseEnabled = null,
73-
$debugLevel = self::LEVEL_NONE
83+
$debugLevel = self::LEVEL_NONE,
84+
$allowSkipped = false
7485
) {
7586
$this->forceGenerate = $forceGenerate;
7687

@@ -89,6 +100,7 @@ private function __construct(
89100
default:
90101
$this->debugLevel = self::LEVEL_DEVELOPER;
91102
}
103+
$this->allowSkipped = $allowSkipped;
92104
}
93105

94106
/**
@@ -99,20 +111,23 @@ private function __construct(
99111
* @param string $phase
100112
* @param boolean $verboseEnabled
101113
* @param string $debugLevel
114+
* @param boolean $allowSkipped
102115
* @return void
116+
* @throws TestFrameworkException
103117
*/
104-
public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
118+
public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped)
105119
{
106120
if (self::$MFTF_APPLICATION_CONTEXT == null) {
107121
self::$MFTF_APPLICATION_CONTEXT =
108-
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
122+
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
109123
}
110124
}
111125

112126
/**
113127
* This function returns an instance of the MftfApplicationConfig which is created once the application starts.
114128
*
115129
* @return MftfApplicationConfig
130+
* @throws TestFrameworkException
116131
*/
117132
public static function getConfig()
118133
{
@@ -157,6 +172,16 @@ public function getDebugLevel()
157172
return $this->debugLevel ?? getenv('MFTF_DEBUG');
158173
}
159174

175+
/**
176+
* Returns a boolean indicating whether mftf is generating skipped tests.
177+
*
178+
* @return boolean
179+
*/
180+
public function allowSkipped()
181+
{
182+
return $this->allowSkipped ?? getenv('ALLOW_SKIPPED');
183+
}
184+
160185
/**
161186
* Returns a string which indicates the phase of mftf execution.
162187
*

‎src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
use Symfony\Component\Console\Output\OutputInterface;
1414
use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
1515
use Magento\FunctionalTestingFramework\Util\TestGenerator;
16+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
1617

1718
class BaseGenerateCommand extends Command
1819
{
@@ -28,6 +29,23 @@ protected function configure()
2829
'r',
2930
InputOption::VALUE_NONE,
3031
'remove previous generated suites and tests'
32+
)->addOption(
33+
"force",
34+
'f',
35+
InputOption::VALUE_NONE,
36+
'force generation and running of tests regardless of Magento Instance Configuration'
37+
)->addOption(
38+
"allowSkipped",
39+
'a',
40+
InputOption::VALUE_NONE,
41+
'Allows MFTF to generate and run skipped tests.'
42+
)->addOption(
43+
'debug',
44+
'd',
45+
InputOption::VALUE_OPTIONAL,
46+
'Run extra validation when generating and running tests. Use option \'none\' to turn off debugging --
47+
added for backward compatibility, will be removed in the next MAJOR release',
48+
MftfApplicationConfig::LEVEL_DEFAULT
3149
);
3250
}
3351

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,6 @@ protected function configure()
3636
'name(s) of specific tests to generate'
3737
)->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
3838
->addOption(
39-
"force",
40-
'f',
41-
InputOption::VALUE_NONE,
42-
'Force generation of tests regardless of Magento Instance Configuration'
43-
)->addOption(
4439
'time',
4540
'i',
4641
InputOption::VALUE_REQUIRED,
@@ -51,13 +46,6 @@ protected function configure()
5146
't',
5247
InputOption::VALUE_REQUIRED,
5348
'A parameter accepting a JSON string used to determine the test configuration'
54-
)->addOption(
55-
'debug',
56-
'd',
57-
InputOption::VALUE_OPTIONAL,
58-
'Run extra validation when generating tests. Use option \'none\' to turn off debugging --
59-
added for backward compatibility, will be removed in the next MAJOR release',
60-
MftfApplicationConfig::LEVEL_DEFAULT
6149
);
6250

6351
parent::configure();
@@ -83,6 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
8371
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
8472
$remove = $input->getOption('remove');
8573
$verbose = $output->isVerbose();
74+
$allowSkipped = $input->getOption('allowSkipped');
8675

8776
if ($json !== null && !json_decode($json)) {
8877
// stop execution if we have failed to properly parse any json passed in by the user
@@ -100,7 +89,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
10089
($debug !== MftfApplicationConfig::LEVEL_NONE));
10190
}
10291

103-
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
92+
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped);
10493

10594
// create our manifest file here
10695
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
@@ -128,18 +117,26 @@ protected function execute(InputInterface $input, OutputInterface $output)
128117
* @param boolean $force
129118
* @param string $debug
130119
* @param boolean $verbose
120+
* @param boolean $allowSkipped
131121
* @return array
132122
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
133123
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
134124
*/
135-
private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
136-
{
125+
private function createTestConfiguration(
126+
string $json,
127+
array $tests,
128+
bool $force,
129+
string $debug,
130+
bool $verbose,
131+
bool $allowSkipped
132+
) {
137133
// set our application configuration so we can references the user options in our framework
138134
MftfApplicationConfig::create(
139135
$force,
140136
MftfApplicationConfig::GENERATION_PHASE,
141137
$verbose,
142-
$debug
138+
$debug,
139+
$allowSkipped
143140
);
144141

145142
$testConfiguration = [];

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,7 @@ protected function configure()
3131
'name',
3232
InputArgument::REQUIRED | InputArgument::IS_ARRAY,
3333
"name of tests to generate and execute"
34-
)->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test")
35-
->addOption(
36-
"force",
37-
'f',
38-
InputOption::VALUE_NONE,
39-
'force generation of tests regardless of Magento Instance Configuration'
40-
)->addOption(
41-
'debug',
42-
'd',
43-
InputOption::VALUE_OPTIONAL,
44-
'Run extra validation when running tests. Use option \'none\' to turn off debugging --
45-
added for backward compatibility, will be removed in the next MAJOR release',
46-
MftfApplicationConfig::LEVEL_DEFAULT
47-
);
34+
)->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test");
4835

4936
parent::configure();
5037
}
@@ -66,6 +53,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
6653
$force = $input->getOption('force');
6754
$remove = $input->getOption('remove');
6855
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
56+
$allowSkipped = $input->getOption('allowSkipped');
6957

7058
if ($skipGeneration and $remove) {
7159
// "skip-generate" and "remove" options cannot be used at the same time
@@ -83,7 +71,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8371
]),
8472
'--force' => $force,
8573
'--remove' => $remove,
86-
'--debug' => $debug
74+
'--debug' => $debug,
75+
'--allowSkipped' => $allowSkipped
8776
];
8877
$command->run(new ArrayInput($args), $output);
8978
}

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,7 @@ class RunTestFailedCommand extends BaseGenerateCommand
5050
protected function configure()
5151
{
5252
$this->setName('run:failed')
53-
->setDescription('Execute a set of tests referenced via failed file')
54-
->addOption(
55-
'debug',
56-
'd',
57-
InputOption::VALUE_OPTIONAL,
58-
'Run extra validation when running failed tests. Use option \'none\' to turn off debugging --
59-
added for backward compatibility, will be removed in the next MAJOR release',
60-
MftfApplicationConfig::LEVEL_DEFAULT
61-
);
53+
->setDescription('Execute a set of tests referenced via failed file');
6254

6355
parent::configure();
6456
}
@@ -76,13 +68,17 @@ protected function configure()
7668
*/
7769
protected function execute(InputInterface $input, OutputInterface $output): int
7870
{
71+
$force = $input->getOption('force');
7972
$debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
73+
$allowSkipped = $input->getOption('allowSkipped');
74+
8075
// Create Mftf Configuration
8176
MftfApplicationConfig::create(
82-
false,
77+
$force,
8378
MftfApplicationConfig::GENERATION_PHASE,
8479
false,
85-
$debug
80+
$debug,
81+
$allowSkipped
8682
);
8783

8884
$testConfiguration = $this->getFailedTestList();
@@ -96,9 +92,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9692
$args = [
9793
'--tests' => $testConfiguration,
9894
'--remove' => true,
99-
'--debug' => $debug
95+
'--debug' => $debug,
96+
'--allowSkipped' => $allowSkipped
10097
];
101-
10298
$command->run(new ArrayInput($args), $output);
10399

104100
$testManifestList = $this->readTestManifestFile();

0 commit comments

Comments
 (0)