diff --git a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt index 130bc1599..3ab774b3d 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt @@ -34,6 +34,6 @@ class ChildExtendedTestNoParentCest */ public function ChildExtendedTestNoParent(AcceptanceTester $I, \Codeception\Scenario $scenario) { - $scenario->skip("This test is skipped"); + $scenario->skip("This test is skipped due to the following issues:\nNo issues have been specified."); } } diff --git a/dev/tests/verification/Resources/MergeSkip.txt b/dev/tests/verification/Resources/MergeSkip.txt new file mode 100644 index 000000000..d6ecff5db --- /dev/null +++ b/dev/tests/verification/Resources/MergeSkip.txt @@ -0,0 +1,34 @@ +skip("This test is skipped due to the following issues:\nIssue5"); + } +} diff --git a/dev/tests/verification/Resources/SkippedTest.txt b/dev/tests/verification/Resources/SkippedTest.txt new file mode 100644 index 000000000..b00fda574 --- /dev/null +++ b/dev/tests/verification/Resources/SkippedTest.txt @@ -0,0 +1,38 @@ +skip("This test is skipped due to the following issues:\nSkippedValue"); + } +} diff --git a/dev/tests/verification/Resources/SkippedTestNoIssues.txt b/dev/tests/verification/Resources/SkippedTestNoIssues.txt new file mode 100644 index 000000000..8d22ca415 --- /dev/null +++ b/dev/tests/verification/Resources/SkippedTestNoIssues.txt @@ -0,0 +1,39 @@ +skip("This test is skipped due to the following issues:\nNo issues have been specified."); + } +} diff --git a/dev/tests/verification/Resources/SkippedTestTwoIssues.txt b/dev/tests/verification/Resources/SkippedTestTwoIssues.txt new file mode 100644 index 000000000..cd140e440 --- /dev/null +++ b/dev/tests/verification/Resources/SkippedTestTwoIssues.txt @@ -0,0 +1,38 @@ +skip("This test is skipped due to the following issues:\nSkippedValue\nSecondSkippedValue"); + } +} diff --git a/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml b/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml index f7997946c..5189b8cf4 100644 --- a/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml @@ -55,4 +55,7 @@ + + + diff --git a/dev/tests/verification/TestModule/Test/SkippedTest.xml b/dev/tests/verification/TestModule/Test/SkippedTest.xml new file mode 100644 index 000000000..bceb619f8 --- /dev/null +++ b/dev/tests/verification/TestModule/Test/SkippedTest.xml @@ -0,0 +1,43 @@ + + + + + + + + + <description value=""/> + <severity value="AVERAGE"/> + <skip> + <issueId value="SkippedValue"/> + </skip> + </annotations> + </test> + <test name="SkippedTestTwoIssues"> + <annotations> + <stories value="skippedMultiple"/> + <title value="skippedMultipleIssuesTest"/> + <description value=""/> + <severity value="AVERAGE"/> + <skip> + <issueId value="SkippedValue"/> + <issueId value="SecondSkippedValue"/> + </skip> + </annotations> + </test> + <test name="SkippedTestNoIssues"> + <annotations> + <stories value="skippedNo"/> + <title value="skippedNoIssuesTest"/> + <description value=""/> + <severity value="AVERAGE"/> + <group value="skip"/> + </annotations> + </test> +</tests> diff --git a/dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml b/dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml index aa8cb375d..912854cc4 100644 --- a/dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml +++ b/dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml @@ -26,4 +26,11 @@ <click stepKey="step10" selector="#step10MergedInResult"/> <actionGroup ref="FunctionalActionGroupWithData" stepKey="step8Merge" after="step7Merge"/> </test> + <test name="MergeSkip"> + <annotations> + <skip> + <issueId value="Issue5"/> + </skip> + </annotations> + </test> </tests> diff --git a/dev/tests/verification/Tests/MergedGenerationTest.php b/dev/tests/verification/Tests/MergedGenerationTest.php index 9ef22fe09..c336cd8d8 100644 --- a/dev/tests/verification/Tests/MergedGenerationTest.php +++ b/dev/tests/verification/Tests/MergedGenerationTest.php @@ -62,4 +62,15 @@ public function testMergeMassViaInsertAfter() { $this->generateAndCompareTest('MergeMassViaInsertAfter'); } + + /** + * Tests generation of a test skipped in merge. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testMergeSkipGeneration() + { + $this->generateAndCompareTest('MergeSkip'); + } } diff --git a/dev/tests/verification/Tests/SkippedGenerationTest.php b/dev/tests/verification/Tests/SkippedGenerationTest.php new file mode 100644 index 000000000..6a65df8e3 --- /dev/null +++ b/dev/tests/verification/Tests/SkippedGenerationTest.php @@ -0,0 +1,44 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +namespace tests\verification\Tests; + +use tests\util\MftfTestCase; + +class SkippedGenerationTest extends MftfTestCase +{ + /** + * Tests skipped test generation. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testSkippedGeneration() + { + $this->generateAndCompareTest('SkippedTest'); + } + + /** + * Tests skipped test with multiple issues generation. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testMultipleSkippedIssuesGeneration() + { + $this->generateAndCompareTest('SkippedTestTwoIssues'); + } + + /** + * Tests skipped test generation with no specified issues. Will be deprecated after MFTF 3.0.0 + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testSkippedNoIssueGeneration() + { + $this->generateAndCompareTest('SkippedTestNoIssues'); + } +} diff --git a/etc/di.xml b/etc/di.xml index eef21e7d7..60faed3a0 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -260,6 +260,7 @@ <item name="/tests/test/annotations/group" xsi:type="string">/tests/test/annotations/group</item> <item name="/tests/test/annotations/env" xsi:type="string">/tests/test/annotations/env</item> <item name="/tests/test/annotations/return" xsi:type="string">/tests/test/annotations/return</item> + <item name="/tests/test/annotations/skip/issueId" xsi:type="string">/tests/test/annotations/skip/issueId</item> </argument> </arguments> </virtualType> diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php index 75970f0dd..71f5ec36b 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php @@ -128,7 +128,10 @@ public function getParentName() */ public function isSkipped() { - if (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) { + // TODO deprecation|deprecate MFTF 3.0.0 remove elseif when group skip is no longer allowed + if (array_key_exists('skip', $this->annotations)) { + return true; + } elseif (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) { return true; } return false; diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php index fa3f675bf..805a010c2 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Test\Util; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; @@ -51,6 +52,7 @@ public function __construct() * @param array $testAnnotations * @param string $filename * @return array + * @throws XmlException */ public function extractAnnotations($testAnnotations, $filename) { @@ -69,9 +71,22 @@ public function extractAnnotations($testAnnotations, $filename) continue; } + if ($annotationKey == "skip") { + $annotationData = $annotationData['issueId']; + $this->validateSkippedIssues($annotationData, $filename); + } + foreach ($annotationData as $annotationValue) { $annotationValues[] = $annotationValue[self::ANNOTATION_VALUE]; } + // TODO deprecation|deprecate MFTF 3.0.0 + if ($annotationKey == "group" && in_array("skip", $annotationValues)) { + LoggingUtil::getInstance()->getLogger(AnnotationExtractor::class)->warning( + "Use of group skip will be deprecated in MFTF 3.0.0. Please update tests to use skip tags.", + ["test" => $filename] + ); + } + $annotationObjects[$annotationKey] = $annotationValues; } @@ -147,6 +162,23 @@ public function validateStoryTitleUniqueness() } } + /** + * Validates that all issueId tags contain a non-empty value + * @param array $issues + * @param string $filename + * @throws XmlException + * @return void + */ + public function validateSkippedIssues($issues, $filename) + { + foreach ($issues as $issueId) { + if (empty($issueId['value'])) { + $message = "issueId for skipped tests cannot be empty. Test: $filename"; + throw new XmlException($message); + } + } + } + /** * This method transforms Magento severity values from Severity annotation * Returns Allure annotation value diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd index 0f8aae936..b65843bae 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd @@ -29,6 +29,7 @@ <xs:element type="annotationType" name="testCaseId" minOccurs="0" maxOccurs="unbounded"/> <xs:element type="annotationType" name="useCaseId" minOccurs="0" maxOccurs="unbounded"/> <xs:element type="annotationType" name="group" minOccurs="0" maxOccurs="unbounded"/> + <xs:element type="skipType" name="skip" minOccurs="0" maxOccurs="unbounded"/> <xs:element type="annotationType" name="return" minOccurs="0" maxOccurs="unbounded"/> </xs:choice> </xs:complexType> @@ -55,6 +56,18 @@ </xs:extension> </xs:simpleContent> </xs:complexType> + <xs:complexType name="skipType"> + <xs:sequence maxOccurs="unbounded"> + <xs:element type="issueType" name="issueId" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="issueType"> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute type="xs:string" name="value" use="required"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> <xs:complexType name="hookType"> <xs:choice minOccurs="0" maxOccurs="unbounded"> <xs:group ref="testTypeTags"/> diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 3929bf041..af39f6ad1 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1561,7 +1561,14 @@ private function generateTestPhp($test) $testAnnotations = $this->generateAnnotationsPhp($test->getAnnotations(), true); $dependencies = 'AcceptanceTester $I'; if ($test->isSkipped()) { - $steps = "\t\t" . '$scenario->skip("This test is skipped");' . "\n"; + $skipString = "This test is skipped due to the following issues:\\n"; + $issues = $test->getAnnotations()['skip'] ?? null; + if (isset($issues)) { + $skipString .= implode("\\n", $issues); + } else { + $skipString .= "No issues have been specified."; + } + $steps = "\t\t" . '$scenario->skip("' . $skipString . '");' . "\n"; $dependencies .= ', \Codeception\Scenario $scenario'; } else { try {