MQE-1236: Allure Does Not Show Results For Tests With Duplicate Title Annotation

KevinBKozan · web-flow · commit 9acb4007d54c · 2018-08-31T13:53:38.000-05:00
- TestCaseID is now prepended to Title annotation (or NONE FOUND)
- Unit/Verification test updates
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
@@ -42,19 +42,19 @@ public function testGetSuiteObject()
 
         $mockGroup1Test1 = $testDataArrayBuilder
             ->withName('group1Test1')
-            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withAnnotations(['group' => [['value' => 'group1']], 'title'=>[['value' => 'group1Test1']]])
             ->withTestActions()
             ->build();
 
         $mockGroup1Test2 = $testDataArrayBuilder
             ->withName('group1Test2')
-            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withAnnotations(['group' => [['value' => 'group1']], 'title'=>[['value' => 'group1Test2']]])
             ->withTestActions()
             ->build();
 
         $mockGroup2Test1 = $testDataArrayBuilder
             ->withName('group2Test1')
-            ->withAnnotations(['group' => [['value' => 'group2']]])
+            ->withAnnotations(['group' => [['value' => 'group2']], 'title'=>[['value' => 'group2Test1']]])
             ->withTestActions()
             ->build();
 
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -121,11 +121,12 @@ public function testGetTestsByGroup()
         // set up mock data with Exclude Test
         $includeTest = (new TestDataArrayBuilder())
             ->withName('includeTest')
-            ->withAnnotations()
+            ->withAnnotations(['group' => [['value' => 'test']], 'title'=>[['value' => 'includeTest']]])
             ->withTestActions()
             ->build();
         $excludeTest = (new TestDataArrayBuilder())
             ->withName('excludeTest')
+            ->withAnnotations(['title'=>[['value' => 'excludeTest']]])
             ->withTestActions()
             ->build();
 
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/AnnotationExtractorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/AnnotationExtractorTest.php
@@ -122,6 +122,89 @@ public function testMissingAnnotations()
         );
     }
 
+    public function testTestCaseIdUniqueness()
+    {
+        // Test Data
+        $firstTestAnnotation = [
+            "nodeName" => "annotations",
+            "features" => [
+                [
+                    "nodeName" => "features",
+                    "value" => "TestFeatures"
+                ]
+            ],
+            "stories" => [
+                [
+                    "nodeName" => "stories",
+                    "value" => "TestStories"
+                ]
+            ],
+            "title" => [
+                [
+                    "nodeName" => "title",
+                    "value" => "TEST TITLE"
+                ]
+            ],
+            "severity" => [
+                [
+                    "nodeName" => "severity",
+                    "value" => "CRITICAL"
+                ]
+            ],
+            "testCaseId" => [
+                [
+                    "nodeName" => "testCaseId",
+                    "value" => "MQE-0001"
+                ]
+            ],
+        ];
+        $secondTestannotation = [
+            "nodeName" => "annotations",
+            "features" => [
+                [
+                    "nodeName" => "features",
+                    "value" => "TestFeatures"
+                ]
+            ],
+            "stories" => [
+                [
+                    "nodeName" => "stories",
+                    "value" => "TestStories"
+                ]
+            ],
+            "title" => [
+                [
+                    "nodeName" => "title",
+                    "value" => "TEST TITLE"
+                ]
+            ],
+            "severity" => [
+                [
+                    "nodeName" => "severity",
+                    "value" => "CRITICAL"
+                ]
+            ],
+            "testCaseId" => [
+                [
+                    "nodeName" => "testCaseId",
+                    "value" => "MQE-0001"
+                ]
+            ],
+        ];
+        // Perform Test
+        $extractor = new AnnotationExtractor();
+        $extractor->extractAnnotations($firstTestAnnotation, "firstTest");
+        $extractor->extractAnnotations($secondTestannotation, "secondTest");
+
+        //Expect Exception
+        $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\XmlException::class);
+        $this->expectExceptionMessage("TestCaseId and Title pairs must be unique:\n\n" .
+            "TestCaseId: 'MQE-0001' Title: 'TEST TITLE' in Tests 'firstTest', 'secondTest'");
+
+        //Trigger Exception
+        $extractor->validateTestCaseIdTitleUniqueness();
+    }
+
     /**
      * After class functionality
      * @return void
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
@@ -43,12 +43,14 @@ public function testGenerateExtendedTest()
 
         $testDataArrayBuilder = new TestDataArrayBuilder();
         $mockSimpleTest = $testDataArrayBuilder
-        ->withName('simpleTest')
-        ->withTestActions($mockActions)
-        ->build();
+            ->withName('simpleTest')
+            ->withAnnotations(['title'=>[['value' => 'simpleTest']]])
+            ->withTestActions($mockActions)
+            ->build();
 
         $mockExtendedTest = $testDataArrayBuilder
             ->withName('extendedTest')
+            ->withAnnotations(['title'=>[['value' => 'extendedTest']]])
             ->withTestReference("simpleTest")
             ->build();
 
@@ -86,12 +88,14 @@ public function testGenerateExtendedWithHooks()
         $testDataArrayBuilder = new TestDataArrayBuilder();
         $mockSimpleTest = $testDataArrayBuilder
             ->withName('simpleTest')
+            ->withAnnotations(['title'=>[['value' => 'simpleTest']]])
             ->withBeforeHook($mockBeforeHooks)
             ->withAfterHook($mockAfterHooks)
             ->build();
 
         $mockExtendedTest = $testDataArrayBuilder
             ->withName('extendedTest')
+            ->withAnnotations(['title'=>[['value' => 'extendedTest']]])
             ->withTestReference("simpleTest")
             ->build();
 
@@ -154,12 +158,14 @@ public function testExtendingExtendedTest()
 
         $mockSimpleTest = $testDataArrayBuilder
             ->withName('simpleTest')
+            ->withAnnotations(['title'=>[['value' => 'simpleTest']]])
             ->withTestActions()
             ->withTestReference("anotherTest")
             ->build();
 
         $mockExtendedTest = $testDataArrayBuilder
             ->withName('extendedTest')
+            ->withAnnotations(['title'=>[['value' => 'extendedTest']]])
             ->withTestReference("simpleTest")
             ->build();
 
diff --git a/dev/tests/verification/Resources/ActionGroupWithDefaultArgumentAndStringSelectorParam.txt b/dev/tests/verification/Resources/ActionGroupWithDefaultArgumentAndStringSelectorParam.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Default Argument Value and Hardcoded Value in Param")
+ * @Title("[NO TESTCASEID]: Action Group With Default Argument Value and Hardcoded Value in Param")
  */
 class ActionGroupWithDefaultArgumentAndStringSelectorParamCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithMultipleParameterSelectorsFromDefaultArgument.txt b/dev/tests/verification/Resources/ActionGroupWithMultipleParameterSelectorsFromDefaultArgument.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Passed Argument Value and Multiple Argument Values in Param")
+ * @Title("[NO TESTCASEID]: Action Group With Passed Argument Value and Multiple Argument Values in Param")
  */
 class ActionGroupWithMultipleParameterSelectorsFromDefaultArgumentCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithNoArguments.txt b/dev/tests/verification/Resources/ActionGroupWithNoArguments.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With No Argument")
+ * @Title("[NO TESTCASEID]: Action Group With No Argument")
  */
 class ActionGroupWithNoArgumentsCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithPassedArgumentAndStringSelectorParam.txt b/dev/tests/verification/Resources/ActionGroupWithPassedArgumentAndStringSelectorParam.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Passed Argument Value and Hardcoded Value in Param")
+ * @Title("[NO TESTCASEID]: Action Group With Passed Argument Value and Hardcoded Value in Param")
  */
 class ActionGroupWithPassedArgumentAndStringSelectorParamCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromDefaultArgument.txt b/dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromDefaultArgument.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Simple Data Usage From Default Argument")
+ * @Title("[NO TESTCASEID]: Action Group With Simple Data Usage From Default Argument")
  */
 class ActionGroupWithSimpleDataUsageFromDefaultArgumentCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromPassedArgument.txt b/dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromPassedArgument.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Simple Data Usage From Passed Argument")
+ * @Title("[NO TESTCASEID]: Action Group With Simple Data Usage From Passed Argument")
  */
 class ActionGroupWithSimpleDataUsageFromPassedArgumentCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromDefaultArgument.txt b/dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromDefaultArgument.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Default Argument Value and Argument Value in Param")
+ * @Title("[NO TESTCASEID]: Action Group With Default Argument Value and Argument Value in Param")
  */
 class ActionGroupWithSingleParameterSelectorFromDefaultArgumentCest
 {
diff --git a/dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromPassedArgument.txt b/dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromPassedArgument.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Action Group With Passed Argument Value and Argument Value in Param")
+ * @Title("[NO TESTCASEID]: Action Group With Passed Argument Value and Argument Value in Param")
  */
 class ActionGroupWithSingleParameterSelectorFromPassedArgumentCest
 {
diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
+ * @Title("[NO TESTCASEID]: A Functional Cest")
  * @group functional
  */
 class BasicFunctionalTestCest
diff --git a/dev/tests/verification/Resources/BasicMergeTest.txt b/dev/tests/verification/Resources/BasicMergeTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("BasicMergeTest")
+ * @Title("[NO TESTCASEID]: BasicMergeTest")
  * @group functional
  * @group mergeTest
  */
diff --git a/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt b/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestAddHooks")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestAddHooks")
  * @group Parent
  */
 class ChildExtendedTestAddHooksCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestMerging.txt b/dev/tests/verification/Resources/ChildExtendedTestMerging.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestMerging")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestMerging")
  * @group Child
  */
 class ChildExtendedTestMergingCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestNoParent")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestNoParent")
  * @group Child
  * @group skip
  */
diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestRemoveAction")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestRemoveAction")
  * @group Child
  */
 class ChildExtendedTestRemoveActionCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestRemoveHookAction")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestRemoveHookAction")
  * @group Child
  */
 class ChildExtendedTestRemoveHookActionCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplace.txt b/dev/tests/verification/Resources/ChildExtendedTestReplace.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestReplace")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestReplace")
  * @group Child
  */
 class ChildExtendedTestReplaceCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt b/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestReplaceHook")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestReplaceHook")
  * @group Child
  */
 class ChildExtendedTestReplaceHookCest
diff --git a/dev/tests/verification/Resources/ExtendingSkippedTest.txt b/dev/tests/verification/Resources/ExtendingSkippedTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ChildExtendedTestSkippedParent")
+ * @Title("[NO TESTCASEID]: ChildExtendedTestSkippedParent")
  * @group Child
  */
 class ExtendingSkippedTestCest
diff --git a/dev/tests/verification/Resources/MergedReferencesTest.txt b/dev/tests/verification/Resources/MergedReferencesTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("MergedReferencesTest")
+ * @Title("[NO TESTCASEID]: MergedReferencesTest")
  * @group functional
  */
 class MergedReferencesTestCest
diff --git a/dev/tests/verification/Resources/ParentExtendedTest.txt b/dev/tests/verification/Resources/ParentExtendedTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("ParentExtendedTest")
+ * @Title("[NO TESTCASEID]: ParentExtendedTest")
  * @group Parent
  */
 class ParentExtendedTestCest
diff --git a/dev/tests/verification/Resources/SkippedTest.txt b/dev/tests/verification/Resources/SkippedTest.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("skippedTest")
+ * @Title("[NO TESTCASEID]: skippedTest")
  * @Description("")
  */
 class SkippedTestCest
diff --git a/dev/tests/verification/Resources/SkippedTestNoIssues.txt b/dev/tests/verification/Resources/SkippedTestNoIssues.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("skippedNoIssuesTest")
+ * @Title("[NO TESTCASEID]: skippedNoIssuesTest")
  * @Description("")
  * @group skip
  */
diff --git a/dev/tests/verification/Resources/SkippedTestTwoIssues.txt b/dev/tests/verification/Resources/SkippedTestTwoIssues.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("skippedMultipleIssuesTest")
+ * @Title("[NO TESTCASEID]: skippedMultipleIssuesTest")
  * @Description("")
  */
 class SkippedTestTwoIssuesCest
diff --git a/dev/tests/verification/Resources/SkippedTestWithHooks.txt b/dev/tests/verification/Resources/SkippedTestWithHooks.txt
@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("skippedTestWithHooks")
+ * @Title("[NO TESTCASEID]: skippedTestWithHooks")
  * @Description("")
  */
 class SkippedTestWithHooksCest
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -151,6 +151,7 @@ private function initTestData()
         $exceptionCollector->throwException();
         
         $testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
+        $testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;`
`15`	`15`	`use Yandex\Allure\Adapter\Annotation\TestCaseId;`
`16`	`16`
`17`	`17`	`/**`
`18`		`- * @Title("Action Group With Default Argument Value and Hardcoded Value in Param")`
	`18`	`+ * @Title("[NO TESTCASEID]: Action Group With Default Argument Value and Hardcoded Value in Param")`
`19`	`19`	`*/`
`20`	`20`	`class ActionGroupWithDefaultArgumentAndStringSelectorParamCest`
`21`	`21`	`{`
Original file line number	Diff line number	Diff line change
`@@ -151,6 +151,7 @@ private function initTestData()`
`151`	`151`	`$exceptionCollector->throwException();`
`152`	`152`
`153`	`153`	`$testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();`
	`154`	`+ $testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();`
`154`	`155`	`}`
`155`	`156`
`156`	`157`	`/**`