MQE-1068: Require Issue ID for Skipped Test

dev/tests/verification/Resources/ChildExtendedTestNoParent.txt

@@ -34,6 +34,6 @@ class ChildExtendedTestNoParentCest
 	 */
 	public function ChildExtendedTestNoParent(AcceptanceTester $I, \Codeception\Scenario $scenario)
 	{
-		$scenario->skip("This test is skipped");
+		$scenario->skip("This test is skipped due to the following issues:\nNo issues have been specified.");
 	}
 }

dev/tests/verification/Resources/MergeSkip.txt

@@ -0,0 +1,34 @@
+<?php
+namespace Magento\AcceptanceTest\_default\Backend;
+
+use Magento\FunctionalTestingFramework\AcceptanceTester;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
+use \Codeception\Util\Locator;
+use Yandex\Allure\Adapter\Annotation\Features;
+use Yandex\Allure\Adapter\Annotation\Stories;
+use Yandex\Allure\Adapter\Annotation\Title;
+use Yandex\Allure\Adapter\Annotation\Description;
+use Yandex\Allure\Adapter\Annotation\Parameter;
+use Yandex\Allure\Adapter\Annotation\Severity;
+use Yandex\Allure\Adapter\Model\SeverityLevel;
+use Yandex\Allure\Adapter\Annotation\TestCaseId;
+
+/**
+ */
+class MergeSkipCest
+{
+	/**
+	 * @Features({"TestModule"})
+	 * @Parameter(name = "AcceptanceTester", value="$I")
+	 * @param AcceptanceTester $I
+	 * @return void
+	 * @throws \Exception
+	 */
+	public function MergeSkip(AcceptanceTester $I, \Codeception\Scenario $scenario)
+	{
+		$scenario->skip("This test is skipped due to the following issues:\nIssue5");
+	}
+}

dev/tests/verification/Resources/SkippedTest.txt

@@ -0,0 +1,38 @@
+<?php
+namespace Magento\AcceptanceTest\_default\Backend;
+
+use Magento\FunctionalTestingFramework\AcceptanceTester;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
+use \Codeception\Util\Locator;
+use Yandex\Allure\Adapter\Annotation\Features;
+use Yandex\Allure\Adapter\Annotation\Stories;
+use Yandex\Allure\Adapter\Annotation\Title;
+use Yandex\Allure\Adapter\Annotation\Description;
+use Yandex\Allure\Adapter\Annotation\Parameter;
+use Yandex\Allure\Adapter\Annotation\Severity;
+use Yandex\Allure\Adapter\Model\SeverityLevel;
+use Yandex\Allure\Adapter\Annotation\TestCaseId;
+
+/**
+ * @Title("skippedTest")
+ * @Description("")
+ */
+class SkippedTestCest
+{
+	/**
+	 * @Stories({"skipped"})
+	 * @Severity(level = SeverityLevel::MINOR)
+	 * @Features({"TestModule"})
+	 * @Parameter(name = "AcceptanceTester", value="$I")
+	 * @param AcceptanceTester $I
+	 * @return void
+	 * @throws \Exception
+	 */
+	public function SkippedTest(AcceptanceTester $I, \Codeception\Scenario $scenario)
+	{
+		$scenario->skip("This test is skipped due to the following issues:\nSkippedValue");
+	}
+}

dev/tests/verification/Resources/SkippedTestNoIssues.txt

@@ -0,0 +1,39 @@
+<?php
+namespace Magento\AcceptanceTest\_default\Backend;
+
+use Magento\FunctionalTestingFramework\AcceptanceTester;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
+use \Codeception\Util\Locator;
+use Yandex\Allure\Adapter\Annotation\Features;
+use Yandex\Allure\Adapter\Annotation\Stories;
+use Yandex\Allure\Adapter\Annotation\Title;
+use Yandex\Allure\Adapter\Annotation\Description;
+use Yandex\Allure\Adapter\Annotation\Parameter;
+use Yandex\Allure\Adapter\Annotation\Severity;
+use Yandex\Allure\Adapter\Model\SeverityLevel;
+use Yandex\Allure\Adapter\Annotation\TestCaseId;
+
+/**
+ * @Title("skippedNoIssuesTest")
+ * @Description("")
+ * @group skip
+ */
+class SkippedTestNoIssuesCest
+{
+	/**
+	 * @Stories({"skippedNo"})
+	 * @Severity(level = SeverityLevel::MINOR)
+	 * @Features({"TestModule"})
+	 * @Parameter(name = "AcceptanceTester", value="$I")
+	 * @param AcceptanceTester $I
+	 * @return void
+	 * @throws \Exception
+	 */
+	public function SkippedTestNoIssues(AcceptanceTester $I, \Codeception\Scenario $scenario)
+	{
+		$scenario->skip("This test is skipped due to the following issues:\nNo issues have been specified.");
+	}
+}

dev/tests/verification/Resources/SkippedTestTwoIssues.txt

@@ -0,0 +1,38 @@
+<?php
+namespace Magento\AcceptanceTest\_default\Backend;
+
+use Magento\FunctionalTestingFramework\AcceptanceTester;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
+use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
+use \Codeception\Util\Locator;
+use Yandex\Allure\Adapter\Annotation\Features;
+use Yandex\Allure\Adapter\Annotation\Stories;
+use Yandex\Allure\Adapter\Annotation\Title;
+use Yandex\Allure\Adapter\Annotation\Description;
+use Yandex\Allure\Adapter\Annotation\Parameter;
+use Yandex\Allure\Adapter\Annotation\Severity;
+use Yandex\Allure\Adapter\Model\SeverityLevel;
+use Yandex\Allure\Adapter\Annotation\TestCaseId;
+
+/**
+ * @Title("skippedMultipleIssuesTest")
+ * @Description("")
+ */
+class SkippedTestTwoIssuesCest
+{
+	/**
+	 * @Stories({"skippedMultiple"})
+	 * @Severity(level = SeverityLevel::MINOR)
+	 * @Features({"TestModule"})
+	 * @Parameter(name = "AcceptanceTester", value="$I")
+	 * @param AcceptanceTester $I
+	 * @return void
+	 * @throws \Exception
+	 */
+	public function SkippedTestTwoIssues(AcceptanceTester $I, \Codeception\Scenario $scenario)
+	{
+		$scenario->skip("This test is skipped due to the following issues:\nSkippedValue\nSecondSkippedValue");
+	}
+}

dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml

@@ -55,4 +55,7 @@
         <click stepKey="clickTwo" selector="#mergeTwo"/>
         <click stepKey="clickThree" selector="#mergeThree"/>
     </test>
+    <test name="MergeSkip">
+        <comment userInput="ThisTestShouldBeSkipped" stepKey="skipComment"/>
+    </test>
 </tests>

dev/tests/verification/TestModule/Test/SkippedTest.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="../../../../../src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd">
+    <test name="SkippedTest">
+        <annotations>
+            <stories value="skipped"/>
+            <title value="skippedTest"/>
+            <description value=""/>
+            <severity value="AVERAGE"/>
+            <skip>
+                <issueId value="SkippedValue"/>
+            </skip>
+        </annotations>
+    </test>
+    <test name="SkippedTestTwoIssues">
+        <annotations>
+            <stories value="skippedMultiple"/>
+            <title value="skippedMultipleIssuesTest"/>
+            <description value=""/>
+            <severity value="AVERAGE"/>
+            <skip>
+                <issueId value="SkippedValue"/>
+                <issueId value="SecondSkippedValue"/>
+            </skip>
+        </annotations>
+    </test>
+    <test name="SkippedTestNoIssues">
+        <annotations>
+            <stories value="skippedNo"/>
+            <title value="skippedNoIssuesTest"/>
+            <description value=""/>
+            <severity value="AVERAGE"/>
+            <group value="skip"/>
+        </annotations>
+    </test>
+</tests>

dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml

@@ -26,4 +26,11 @@
         <click stepKey="step10" selector="#step10MergedInResult"/>
         <actionGroup ref="FunctionalActionGroupWithData" stepKey="step8Merge" after="step7Merge"/>
     </test>
+    <test name="MergeSkip">
+        <annotations>
+            <skip>
+                <issueId value="Issue5"/>
+            </skip>
+        </annotations>
+    </test>
 </tests>

dev/tests/verification/Tests/MergedGenerationTest.php

@@ -62,4 +62,15 @@ public function testMergeMassViaInsertAfter()
     {
         $this->generateAndCompareTest('MergeMassViaInsertAfter');
     }
+
+    /**
+     * Tests generation of a test skipped in merge.
+     *
+     * @throws \Exception
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testMergeSkipGeneration()
+    {
+        $this->generateAndCompareTest('MergeSkip');
+    }
 }

dev/tests/verification/Tests/SkippedGenerationTest.php

@@ -0,0 +1,44 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace tests\verification\Tests;
+
+use tests\util\MftfTestCase;
+
+class SkippedGenerationTest extends MftfTestCase
+{
+    /**
+     * Tests skipped test generation.
+     *
+     * @throws \Exception
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testSkippedGeneration()
+    {
+        $this->generateAndCompareTest('SkippedTest');
+    }
+
+    /**
+     * Tests skipped test with multiple issues generation.
+     *
+     * @throws \Exception
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testMultipleSkippedIssuesGeneration()
+    {
+        $this->generateAndCompareTest('SkippedTestTwoIssues');
+    }
+
+    /**
+     * Tests skipped test generation with no specified issues. Will be deprecated after MFTF 3.0.0
+     *
+     * @throws \Exception
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testSkippedNoIssueGeneration()
+    {
+        $this->generateAndCompareTest('SkippedTestNoIssues');
+    }
+}

etc/di.xml

@@ -260,6 +260,7 @@
                 <item name="/tests/test/annotations/group" xsi:type="string">/tests/test/annotations/group</item>
                 <item name="/tests/test/annotations/env" xsi:type="string">/tests/test/annotations/env</item>
                 <item name="/tests/test/annotations/return" xsi:type="string">/tests/test/annotations/return</item>
+                <item name="/tests/test/annotations/skip/issueId" xsi:type="string">/tests/test/annotations/skip/issueId</item>
             </argument>
         </arguments>
     </virtualType>

src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

@@ -128,7 +128,10 @@ public function getParentName()
      */
     public function isSkipped()
     {
-        if (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
+        // TODO deprecation|deprecate MFTF 3.0.0 remove elseif when group skip is no longer allowed
+        if (array_key_exists('skip', $this->annotations)) {
+            return true;
+        } elseif (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
             return true;
         }
         return false;

src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

@@ -6,6 +6,7 @@
 
 namespace Magento\FunctionalTestingFramework\Test\Util;
 
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 
@@ -51,6 +52,7 @@ public function __construct()
      * @param array  $testAnnotations
      * @param string $filename
      * @return array
+     * @throws XmlException
      */
     public function extractAnnotations($testAnnotations, $filename)
     {
@@ -69,9 +71,22 @@ public function extractAnnotations($testAnnotations, $filename)
                 continue;
             }
 
+            if ($annotationKey == "skip") {
+                $annotationData = $annotationData['issueId'];
+                $this->validateSkippedIssues($annotationData, $filename);
+            }
+
             foreach ($annotationData as $annotationValue) {
                 $annotationValues[] = $annotationValue[self::ANNOTATION_VALUE];
             }
+            // TODO deprecation|deprecate MFTF 3.0.0
+            if ($annotationKey == "group" && in_array("skip", $annotationValues)) {
+                LoggingUtil::getInstance()->getLogger(AnnotationExtractor::class)->warning(
+                    "Use of group skip will be deprecated in MFTF 3.0.0. Please update tests to use skip tags.",
+                    ["test" => $filename]
+                );
+            }
+
             $annotationObjects[$annotationKey] = $annotationValues;
         }
 
@@ -147,6 +162,23 @@ public function validateStoryTitleUniqueness()
         }
     }
 
+    /**
+     * Validates that all issueId tags contain a non-empty value
+     * @param array  $issues
+     * @param string $filename
+     * @throws XmlException
+     * @return void
+     */
+    public function validateSkippedIssues($issues, $filename)
+    {
+        foreach ($issues as $issueId) {
+            if (empty($issueId['value'])) {
+                $message = "issueId for skipped tests cannot be empty. Test: $filename";
+                throw new XmlException($message);
+            }
+        }
+    }
+
     /**
      * This method transforms Magento severity values from Severity annotation
      * Returns Allure annotation value

src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd

@@ -29,6 +29,7 @@
             <xs:element type="annotationType" name="testCaseId" minOccurs="0" maxOccurs="unbounded"/>
             <xs:element type="annotationType" name="useCaseId" minOccurs="0" maxOccurs="unbounded"/>
             <xs:element type="annotationType" name="group" minOccurs="0" maxOccurs="unbounded"/>
+            <xs:element type="skipType" name="skip" minOccurs="0" maxOccurs="unbounded"/>
             <xs:element type="annotationType" name="return" minOccurs="0" maxOccurs="unbounded"/>
         </xs:choice>
     </xs:complexType>
@@ -55,6 +56,18 @@
             </xs:extension>
         </xs:simpleContent>
     </xs:complexType>
+    <xs:complexType name="skipType">
+        <xs:sequence maxOccurs="unbounded">
+            <xs:element type="issueType" name="issueId" maxOccurs="unbounded"/>
+        </xs:sequence>
+    </xs:complexType>
+    <xs:complexType name="issueType">
+        <xs:simpleContent>
+            <xs:extension base="xs:string">
+                <xs:attribute type="xs:string" name="value" use="required"/>
+            </xs:extension>
+        </xs:simpleContent>
+    </xs:complexType>
     <xs:complexType name="hookType">
         <xs:choice minOccurs="0" maxOccurs="unbounded">
             <xs:group ref="testTypeTags"/>