diff --git a/etc/config/codeception.dist.yml b/etc/config/codeception.dist.yml index 273ede0b0..57582f1bd 100755 --- a/etc/config/codeception.dist.yml +++ b/etc/config/codeception.dist.yml @@ -15,6 +15,7 @@ extensions: - Codeception\Extension\RunFailed - Magento\FunctionalTestingFramework\Extension\TestContextExtension - Magento\FunctionalTestingFramework\Allure\Adapter\MagentoAllureAdapter + - Magento\FunctionalTestingFramework\Extension\PageReadinessExtension config: Yandex\Allure\Adapter\AllureAdapter: deletePreviousResults: true @@ -23,5 +24,15 @@ extensions: - env - zephyrId - useCaseId + Magento\FunctionalTestingFramework\Extension\PageReadinessExtension: + driver: \Magento\FunctionalTestingFramework\Module\MagentoWebDriver + timeout: 30 + resetFailureThreshold: 3 + readinessMetrics: + - \Magento\FunctionalTestingFramework\Extension\ReadinessMetrics\DocumentReadyState + - \Magento\FunctionalTestingFramework\Extension\ReadinessMetrics\JQueryAjaxRequests + - \Magento\FunctionalTestingFramework\Extension\ReadinessMetrics\PrototypeAjaxRequests + - \Magento\FunctionalTestingFramework\Extension\ReadinessMetrics\RequireJsDefinitions + - \Magento\FunctionalTestingFramework\Extension\ReadinessMetrics\MagentoLoadingMasks params: - .env \ No newline at end of file diff --git a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php new file mode 100644 index 000000000..393d03644 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php @@ -0,0 +1,270 @@ + 'beforeTest', + Events::STEP_BEFORE => 'beforeStep' + ]; + + /** + * List of action types that should bypass metric checks + * shouldSkipCheck() also checks for the 'Comment' step type, which doesn't follow the $step->getAction() pattern + * + * @var array + */ + private $ignoredActions = [ + 'saveScreenshot', + 'wait' + ]; + + /** + * @var Logger + */ + private $logger; + + /** + * Logger verbosity + * + * @var boolean + */ + private $verbose; + + /** + * Array of readiness metrics, initialized during beforeTest event + * + * @var AbstractMetricCheck[] + */ + private $readinessMetrics; + + /** + * The name of the active test + * + * @var string + */ + private $testName; + + /** + * The current URI of the active page + * + * @var string + */ + private $uri; + + /** + * Initialize local vars + * + * @return void + * @throws \Exception + */ + public function _initialize() + { + $this->logger = LoggingUtil::getInstance()->getLogger(get_class($this)); + $this->verbose = MftfApplicationConfig::getConfig()->verboseEnabled(); + } + + /** + * WebDriver instance to use to execute readiness metric checks + * + * @return WebDriver + * @throws ModuleRequireException + */ + public function getDriver() + { + return $this->getModule($this->config['driver']); + } + + /** + * Initialize the readiness metrics for the test + * + * @param \Codeception\Event\TestEvent $e + * @return void + */ + public function beforeTest(TestEvent $e) + { + if (isset($this->config['resetFailureThreshold'])) { + $failThreshold = intval($this->config['resetFailureThreshold']); + } else { + $failThreshold = 3; + } + + $this->testName = $e->getTest()->getMetadata()->getName(); + $this->uri = null; + + $metrics = []; + foreach ($this->config['readinessMetrics'] as $metricClass) { + $metrics[] = new $metricClass($this, $failThreshold); + } + + $this->readinessMetrics = $metrics; + } + + /** + * Waits for busy page flags to disappear before executing a step + * + * @param StepEvent $e + * @return void + * @throws \Exception + */ + public function beforeStep(StepEvent $e) + { + $step = $e->getStep(); + if ($this->shouldSkipCheck($step)) { + return; + } + + $this->checkForNewPage($step); + + // todo: Implement step parameter to override global timeout configuration + if (isset($this->config['timeout'])) { + $timeout = intval($this->config['timeout']); + } else { + $timeout = $this->getDriver()->_getConfig()['pageload_timeout']; + } + + $metrics = $this->readinessMetrics; + + try { + $this->getDriver()->webDriver->wait($timeout)->until( + function () use ($metrics) { + $passing = true; + + /** @var AbstractMetricCheck $metric */ + foreach ($metrics as $metric) { + try { + if (!$metric->runCheck()) { + $passing = false; + } + } catch (UnexpectedAlertOpenException $exception) { + } + } + return $passing; + } + ); + } catch (TimeoutException $exception) { + } + + /** @var AbstractMetricCheck $metric */ + foreach ($metrics as $metric) { + $metric->finalizeForStep($step); + } + } + + /** + * Check if the URI has changed and reset metric tracking if so + * + * @param Step $step + * @return void + */ + private function checkForNewPage($step) + { + try { + $currentUri = $this->getDriver()->_getCurrentUri(); + + if ($this->uri !== $currentUri) { + $this->logDebug( + 'Page URI changed; resetting readiness metric failure tracking', + [ + 'step' => $step->__toString(), + 'newUri' => $currentUri + ] + ); + + /** @var AbstractMetricCheck $metric */ + foreach ($this->readinessMetrics as $metric) { + $metric->resetTracker(); + } + + $this->uri = $currentUri; + } + } catch (\Exception $e) { + $this->logDebug('Could not retrieve current URI', ['step' => $step->__toString()]); + } + } + + /** + * Gets the active page URI from the start of the most recent step + * + * @return string + */ + public function getUri() + { + return $this->uri; + } + + /** + * Gets the name of the active test + * + * @return string + */ + public function getTestName() + { + return $this->testName; + } + + /** + * Should the given step bypass the readiness checks + * todo: Implement step parameter to bypass specific metrics (or all) instead of basing on action type + * + * @param Step $step + * @return boolean + */ + private function shouldSkipCheck($step) + { + if ($step instanceof Step\Comment || in_array($step->getAction(), $this->ignoredActions)) { + return true; + } + return false; + } + + /** + * If verbose, log the given message to logger->debug including test context information + * + * @param string $message + * @param array $context + * @return void + */ + private function logDebug($message, $context = []) + { + if ($this->verbose) { + $logContext = [ + 'test' => $this->testName, + 'uri' => $this->uri + ]; + foreach ($this->readinessMetrics as $metric) { + $logContext[$metric->getName()] = $metric->getStoredValue(); + $logContext[$metric->getName() . '.failCount'] = $metric->getFailureCount(); + } + $context = array_merge($logContext, $context); + $this->logger->info($message, $context); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/AbstractMetricCheck.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/AbstractMetricCheck.php new file mode 100644 index 000000000..c5dd7891b --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/AbstractMetricCheck.php @@ -0,0 +1,359 @@ +extension = $extension; + $this->logger = LoggingUtil::getInstance()->getLogger(get_class($this)); + $this->verbose = MftfApplicationConfig::getConfig()->verboseEnabled(); + + // If the clearFailureOnPage() method is overridden, use the configured failure threshold + // If not, the default clearFailureOnPage() method does nothing so don't worry about resetting failures + $reflector = new \ReflectionMethod($this, 'clearFailureOnPage'); + if ($reflector->getDeclaringClass()->getName() === get_class($this)) { + $this->resetFailureThreshold = $resetFailureThreshold; + } else { + $this->resetFailureThreshold = -1; + } + + $this->resetTracker(); + } + + /** + * Does the given value pass the readiness metric + * + * @param mixed $value + * @return boolean + */ + abstract protected function doesMetricPass($value); + + /** + * Retrieve the active value for the metric to check from the page + * + * @return mixed + * @throws UnexpectedAlertOpenException + */ + abstract protected function fetchValueFromPage(); + + /** + * Override this method to reset the actual state of the page to make the metric pass + * This method is called when too many identical failures were encountered in a row + * + * @return void + */ + protected function clearFailureOnPage() + { + return; + } + + /** + * Get the base class name of the metric implementation + * + * @return string + */ + public function getName() + { + $clazz = get_class($this); + $namespaceBreak = strrpos($clazz, '\\'); + if ($namespaceBreak !== false) { + $clazz = substr($clazz, $namespaceBreak + 1); + } + return $clazz; + } + + /** + * Fetches a new value for the metric and checks if it passes, clearing the failure tracking if so + * + * Even on a success, the readiness check will continue to be run until all metrics pass at the same time in order + * to catch cases where a slow request of one metric can trigger calls for other metrics that were previously + * thought ready + * + * @return boolean + * @throws UnexpectedAlertOpenException + */ + public function runCheck() + { + if ($this->doesMetricPass($this->getCurrentValue(true))) { + $this->setTracker($this->getCurrentValue(), 0); + return true; + } + + return false; + } + + /** + * Update the state of the metric including tracked failure state and checking if a failing value is stuck and + * needs to be reset so future checks can be accurate + * + * Called when the readiness check is finished (either all metrics pass or the check has timed out) + * + * @param Step $step + * @return void + */ + public function finalizeForStep($step) + { + try { + $currentValue = $this->getCurrentValue(); + } catch (UnexpectedAlertOpenException $exception) { + $this->debugLog( + 'An alert is open, bypassing javascript-based metric check', + ['step' => $step->__toString()] + ); + return; + } + + if ($this->doesMetricPass($currentValue)) { + $this->setTracker($currentValue, 0); + } else { + // If failure happened on the same value as before, increment the fail count, otherwise set at 1 + if (!isset($this->storedValue) || $currentValue !== $this->getStoredValue()) { + $failCount = 1; + } else { + $failCount = $this->getFailureCount() + 1; + } + $this->setTracker($currentValue, $failCount); + + $this->errorLog('Failed readiness check', ['step' => $step->__toString()]); + + if ($this->resetFailureThreshold >= 0 && $failCount >= $this->resetFailureThreshold) { + $this->debugLog( + 'Too many failures, assuming metric is stuck and resetting state', + ['step' => $step->__toString()] + ); + $this->resetMetric(); + } + } + } + + /** + * Helper function to retrieve the driver being used to run the test + * + * @return WebDriver + * @throws ModuleRequireException + */ + protected function getDriver() + { + return $this->extension->getDriver(); + } + + /** + * Helper function to execute javascript code, see WebDriver::executeJs for more information + * + * @param string $script + * @param array $arguments + * @return mixed + * @throws UnexpectedAlertOpenException + * @throws ModuleRequireException + */ + protected function executeJs($script, $arguments = []) + { + return $this->extension->getDriver()->executeJS($script, $arguments); + } + + /** + * Gets the current state of the given variable + * Fetches an updated value if not known or $refresh is true + * + * @param boolean $refresh + * @return mixed + * @throws UnexpectedAlertOpenException + */ + private function getCurrentValue($refresh = false) + { + if ($refresh) { + unset($this->currentValue); + } + if (!isset($this->currentValue)) { + $this->currentValue = $this->fetchValueFromPage(); + } + return $this->currentValue; + } + + /** + * Returns the value of the given variable for the previous check + * + * @return mixed + */ + public function getStoredValue() + { + return $this->storedValue; + } + + /** + * The current count of sequential identical failures + * Used to detect potentially stuck metrics + * + * @return integer + */ + public function getFailureCount() + { + return $this->failCount; + } + + /** + * Update the state of the page to pass the metric and clear the saved failure state + * Called when a failure is found to be stuck + * + * @return void + */ + private function resetMetric() + { + $this->clearFailureOnPage(); + $this->resetTracker(); + } + + /** + * Tracks the most recent value and the number of identical failures in a row + * + * @param mixed $value + * @param integer $failCount + * @return void + */ + public function setTracker($value, $failCount) + { + unset($this->currentValue); + $this->storedValue = $value; + $this->failCount = $failCount; + } + + /** + * Resets the tracked metric values on a new page or stuck failure + * + * @return void + */ + public function resetTracker() + { + unset($this->currentValue); + unset($this->storedValue); + $this->failCount = 0; + } + + /** + * Log the given message to logger->error including context information + * + * @param string $message + * @param array $context + * @return void + */ + protected function errorLog($message, $context = []) + { + $context = array_merge($this->getLogContext(), $context); + $this->logger->error($message, $context); + } + + /** + * Log the given message to logger->info including context information + * + * @param string $message + * @param array $context + * @return void + */ + protected function infoLog($message, $context = []) + { + $context = array_merge($this->getLogContext(), $context); + $this->logger->info($message, $context); + } + + /** + * If verbose, log the given message to logger->debug including context information + * + * @param string $message + * @param array $context + * @return void + */ + protected function debugLog($message, $context = []) + { + if ($this->verbose) { + $context = array_merge($this->getLogContext(), $context); + $this->logger->debug($message, $context); + } + } + + /** + * Base context information to include in all log messages: test name, current URI, metric state + * Reports most recent stored value, not current value, so call setTracker() first to update + * + * @return array + */ + private function getLogContext() + { + return [ + 'test' => $this->extension->getTestName(), + 'uri' => $this->extension->getUri(), + $this->getName() => $this->getStoredValue(), + $this->getName() . '.failCount' => $this->getFailureCount() + ]; + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/DocumentReadyState.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/DocumentReadyState.php new file mode 100644 index 000000000..26cf91aa7 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/DocumentReadyState.php @@ -0,0 +1,43 @@ +executeJS('return document.readyState;'); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/JQueryAjaxRequests.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/JQueryAjaxRequests.php new file mode 100644 index 000000000..c005923d3 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/JQueryAjaxRequests.php @@ -0,0 +1,58 @@ +executeJS( + 'if (!!window.jQuery) { + return window.jQuery.active; + } + return 0;' + ) + ); + } + + /** + * Active request count can get stuck above zero if an exception is thrown during a callback, causing the + * ajax handler method to fail before decrementing the request count + * + * @return void + * @throws UnexpectedAlertOpenException + */ + protected function clearFailureOnPage() + { + $this->executeJS('if (!!window.jQuery) { window.jQuery.active = 0; };'); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/MagentoLoadingMasks.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/MagentoLoadingMasks.php new file mode 100644 index 000000000..4f15524ba --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/MagentoLoadingMasks.php @@ -0,0 +1,54 @@ +getDriver()->webDriver->findElements($driverLocator); + foreach ($maskElements as $element) { + try { + if ($element->isDisplayed()) { + return "$maskLocator : " . $element ->getID(); + } + } catch (NoSuchElementException $e) { + } catch (StaleElementReferenceException $e) { + } + } + } + return null; + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/PrototypeAjaxRequests.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/PrototypeAjaxRequests.php new file mode 100644 index 000000000..2fc8f70cb --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/PrototypeAjaxRequests.php @@ -0,0 +1,58 @@ +executeJS( + 'if (!!window.Prototype) { + return window.Ajax.activeRequestCount; + } + return 0;' + ) + ); + } + + /** + * Active request count can get stuck above zero if an exception is thrown during a callback, causing the + * ajax handler method to fail before decrementing the request count + * + * @return void + * @throws UnexpectedAlertOpenException + */ + protected function clearFailureOnPage() + { + $this->executeJS('if (!!window.Prototype) { window.Ajax.activeRequestCount = 0; };'); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/RequireJsDefinitions.php b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/RequireJsDefinitions.php new file mode 100644 index 000000000..6df470123 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/ReadinessMetrics/RequireJsDefinitions.php @@ -0,0 +1,60 @@ +executeJS($script); + if ($moduleInProgress === 'null') { + $moduleInProgress = null; + } + return $moduleInProgress; + } +}