MQE-1006: Handling secure/sensitive data in MFTF test

dev/tests/_bootstrap.php

@@ -31,7 +31,8 @@
 \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::create(
     true,
     \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::GENERATION_PHASE,
-    true
+    true,
+    false
 );
 
 // Load needed framework env params

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionGroupObjectExtractorTest.php

@@ -0,0 +1,70 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace tests\unit\Magento\FunctionalTestFramework\Test\Util;
+
+use Magento\FunctionalTestingFramework\Test\Util\ActionGroupObjectExtractor;
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use tests\unit\Util\TestLoggingUtil;
+
+class ActionGroupObjectExtractorTest extends MagentoTestCase
+{
+    /** @var  ActionGroupObjectExtractor */
+    private $testActionGroupObjectExtractor;
+
+    /**
+     * Setup method
+     */
+    public function setUp()
+    {
+        $this->testActionGroupObjectExtractor = new ActionGroupObjectExtractor();
+        TestLoggingUtil::getInstance()->setMockLoggingUtil();
+    }
+
+    /**
+     * Tests basic action object extraction with an empty stepKey
+     */
+    public function testEmptyStepKey()
+    {
+        $this->expectExceptionMessage(
+            "StepKeys cannot be empty.	Action='sampleAction' in Action Group filename.xml"
+        );
+        $this->testActionGroupObjectExtractor->extractActionGroup($this->createBasicActionObjectArray(""));
+    }
+
+    /**
+     * Utility function to return mock parser output for testing extraction into ActionObjects.
+     *
+     * @param string $stepKey
+     * @param string $actionGroup
+     * @param string $filename
+     * @return array
+     */
+    private function createBasicActionObjectArray(
+        $stepKey = 'testAction1',
+        $actionGroup = "actionGroup",
+        $filename = "filename.xml"
+    ) {
+        $baseArray = [
+            'nodeName' => 'actionGroup',
+            'name' => $actionGroup,
+            'filename' => $filename,
+            $stepKey => [
+                "nodeName" => "sampleAction",
+                "stepKey" => $stepKey,
+                "someAttribute" => "someAttributeValue"
+            ]
+        ];
+        return $baseArray;
+    }
+
+    /**
+     * clean up function runs after all tests
+     */
+    public static function tearDownAfterClass()
+    {
+        TestLoggingUtil::getInstance()->clearMockLoggingUtil();
+    }
+}

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionObjectExtractorTest.php

@@ -50,7 +50,7 @@ public function testInvalidMergeOrderReference()
         } catch (\Exception $e) {
             TestLoggingUtil::getInstance()->validateMockLogStatement(
                 'error',
-                'Line 103: Invalid ordering configuration in test',
+                'Line 108: Invalid ordering configuration in test',
                 [
                     'test' => 'TestWithSelfReferencingStepKey',
                     'stepKey' => ['invalidTestAction1']
@@ -89,6 +89,15 @@ public function testAmbiguousMergeOrderReference()
         );
     }
 
+    /**
+     * Tests basic action object extraction with an empty stepKey
+     */
+    public function testEmptyStepKey()
+    {
+        $this->expectExceptionMessage("StepKeys cannot be empty.	Action='sampleAction'");
+        $this->testActionObjectExtractor->extractActions($this->createBasicActionObjectArray(""));
+    }
+
     /**
      * Utility function to return mock parser output for testing extraction into ActionObjects.
      *

dev/tests/util/MftfTestCase.php

@@ -5,13 +5,20 @@
  */
 namespace tests\util;
 
+use Magento\FunctionalTestingFramework\ObjectManager;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 use PHPUnit\Framework\TestCase;
 
 abstract class MftfTestCase extends TestCase
 {
-    const RESOURCES_PATH = __DIR__ . '/../verification/Resources';
+    const RESOURCES_PATH = __DIR__ .
+    DIRECTORY_SEPARATOR .
+    '..' .
+    DIRECTORY_SEPARATOR .
+    'verification' .
+    DIRECTORY_SEPARATOR .
+    'Resources';
 
     /**
      * Private function which takes a test name, generates the test and compares with a correspondingly named txt file
@@ -37,4 +44,57 @@ public function generateAndCompareTest($testName)
             $cestFile
         );
     }
-}
+
+    /**
+     * Private function which attempts to generate tests given an invalid shcema of a various type
+     *
+     * @param string[] $fileContents
+     * @param string $objectType
+     * @param string $expectedError
+     * @throws \Exception
+     */
+    public function validateSchemaErrorWithTest($fileContents, $objectType ,$expectedError)
+    {
+        $this->clearHandler();
+        $fullTestModulePath = TESTS_MODULE_PATH .
+            DIRECTORY_SEPARATOR .
+            'TestModule' .
+            DIRECTORY_SEPARATOR .
+            $objectType .
+            DIRECTORY_SEPARATOR;
+
+        foreach ($fileContents as $fileName => $fileContent) {
+            $tempFile = $fullTestModulePath . $fileName;
+            $handle = fopen($tempFile, 'w') or die('Cannot open file:  ' . $tempFile);
+            fwrite($handle, $fileContent);
+            fclose($handle);
+        }
+        try {
+            $this->expectExceptionMessage($expectedError);
+            TestObjectHandler::getInstance()->getObject("someTest");
+        } finally {
+            foreach (array_keys($fileContents) as $fileName) {
+                unlink($fullTestModulePath . $fileName);
+            }
+            $this->clearHandler();
+        }
+    }
+
+    /**
+     * Clears test handler and object manager to force recollection of test data
+     *
+     * @throws \Exception
+     */
+    private function clearHandler()
+    {
+        // clear test object handler to force recollection of test data
+        $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
+        $property->setAccessible(true);
+        $property->setValue(null);
+
+        // clear test object handler to force recollection of test data
+        $property = new \ReflectionProperty(ObjectManager::class, 'instance');
+        $property->setAccessible(true);
+        $property->setValue(null);
+    }
+}

dev/tests/verification/TestModule/Test/PageReplacementTest.xml

@@ -22,7 +22,4 @@
         <amOnPage stepKey="oneParamAdminPageString" url="{{AdminOneParamPage.url('StringLiteral')}}"/>
         <amOnUrl stepKey="onExternalPage" url="{{ExternalPage.url}}"/>
     </test>
-    <test name="ExternalPageTestBadReference">
-        <amOnPage stepKey="onExternalPage" url="{{ExternalPage.url}}"/>
-    </test>
 </tests>

dev/tests/verification/Tests/SchemaValidationTest.php

@@ -0,0 +1,42 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace tests\verification\Tests;
+
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use tests\util\MftfTestCase;
+use AspectMock\Test as AspectMock;
+
+class SchemaValidationTest extends MftfTestCase
+{
+    /**
+     * Test generation of a test referencing an action group with no arguments
+     *
+     * @throws \Exception
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testInvalidTestSchema()
+    {
+        AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]);
+        $testFile = ['testFile.xml' => "<tests><test name='testName'><annotations>a</annotations></test></tests>"];
+        $expectedError = TESTS_MODULE_PATH .
+            DIRECTORY_SEPARATOR .
+            "TestModule" .
+            DIRECTORY_SEPARATOR .
+            "Test" .
+            DIRECTORY_SEPARATOR .
+            "testFile.xml";
+        $this->validateSchemaErrorWithTest($testFile, 'Test', $expectedError);
+    }
+
+    /**
+     * After method functionality
+     * @return void
+     */
+    protected function tearDown()
+    {
+        AspectMock::clean();
+    }
+}

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -34,6 +34,13 @@ class MftfApplicationConfig
      */
     private $verboseEnabled;
 
+    /**
+     * Determines whether the user would like to execute mftf in a verbose run.
+     *
+     * @var bool
+     */
+    private $debugEnabled;
+
     /**
      * MftfApplicationConfig Singelton Instance
      *
@@ -47,10 +54,15 @@ class MftfApplicationConfig
      * @param bool $forceGenerate
      * @param string $phase
      * @param bool $verboseEnabled
+     * @param bool $debugEnabled
      * @throws TestFrameworkException
      */
-    private function __construct($forceGenerate = false, $phase = self::EXECUTION_PHASE, $verboseEnabled = null)
-    {
+    private function __construct(
+        $forceGenerate = false,
+        $phase = self::EXECUTION_PHASE,
+        $verboseEnabled = null,
+        $debugEnabled = null
+    ) {
         $this->forceGenerate = $forceGenerate;
 
         if (!in_array($phase, self::MFTF_PHASES)) {
@@ -59,6 +71,7 @@ private function __construct($forceGenerate = false, $phase = self::EXECUTION_PH
 
         $this->phase = $phase;
         $this->verboseEnabled = $verboseEnabled;
+        $this->debugEnabled = $debugEnabled;
     }
 
     /**
@@ -68,12 +81,14 @@ private function __construct($forceGenerate = false, $phase = self::EXECUTION_PH
      * @param bool $forceGenerate
      * @param string $phase
      * @param bool $verboseEnabled
+     * @param bool $debugEnabled
      * @return void
      */
-    public static function create($forceGenerate, $phase, $verboseEnabled)
+    public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
     {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
-            self::$MFTF_APPLICATION_CONTEXT = new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled);
+            self::$MFTF_APPLICATION_CONTEXT =
+                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
         }
     }
 
@@ -115,6 +130,17 @@ public function verboseEnabled()
         return $this->verboseEnabled ?? getenv('MFTF_DEBUG');
     }
 
+    /**
+     * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
+     * with some extra error messaging to be run
+     *
+     * @return bool
+     */
+    public function debugEnabled()
+    {
+        return $this->debugEnabled ?? getenv('MFTF_DEBUG');
+    }
+
     /**
      * Returns a string which indicates the phase of mftf execution.
      *

src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php

@@ -157,17 +157,14 @@ protected function readFiles($fileList)
                 } else {
                     $configMerger->merge($content);
                 }
+                if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+                    $this->validateSchema($configMerger, $fileList->getFilename());
+                }
             } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
                 throw new \Exception("Invalid XML in file " . $fileList->getFilename() . ":\n" . $e->getMessage());
             }
         }
-        if ($this->validationState->isValidationRequired()) {
-            $errors = [];
-            if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
-                $message = "Invalid Document \n";
-                throw new \Exception($message . implode("\n", $errors));
-            }
-        }
+        $this->validateSchema($configMerger);
 
         $output = [];
         if ($configMerger) {
@@ -220,4 +217,23 @@ protected function verifyFileEmpty($content, $fileName)
         }
         return true;
     }
+
+    /**
+     * Validate read xml against expected schema
+     *
+     * @param string $configMerger
+     * @param string $filename
+     * @throws \Exception
+     * @return void
+     */
+    protected function validateSchema($configMerger, $filename = null)
+    {
+        if ($this->validationState->isValidationRequired()) {
+            $errors = [];
+            if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
+                $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
+                throw new \Exception($message . implode("\n", $errors));
+            }
+        }
+    }
 }

src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php

@@ -6,6 +6,7 @@
 
 namespace Magento\FunctionalTestingFramework\Config\Reader;
 
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
 use Magento\FunctionalTestingFramework\Util\Iterator\File;
 
@@ -21,7 +22,6 @@ class MftfFilesystem extends \Magento\FunctionalTestingFramework\Config\Reader\F
     public function readFiles($fileList)
     {
         $exceptionCollector = new ExceptionCollector();
-        $errors = [];
         /** @var \Magento\FunctionalTestingFramework\Test\Config\Dom $configMerger */
         $configMerger = null;
         foreach ($fileList as $key => $content) {
@@ -40,17 +40,15 @@ public function readFiles($fileList)
                 } else {
                     $configMerger->merge($content, $fileList->getFilename(), $exceptionCollector);
                 }
+                if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+                    $this->validateSchema($configMerger, $fileList->getFilename());
+                }
             } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
                 throw new \Exception("Invalid XML in file " . $key . ":\n" . $e->getMessage());
             }
         }
         $exceptionCollector->throwException();
-        if ($this->validationState->isValidationRequired()) {
-            if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
-                $message = "Invalid Document \n";
-                throw new \Exception($message . implode("\n", $errors));
-            }
-        }
+        $this->validateSchema($configMerger, $fileList->getFilename());
 
         $output = [];
         if ($configMerger) {