/** * Run entire test suite of scenarios */ public function run() { $this->_reportFiles = []; $scenarios = $this->_getOptimizedScenarioList(); foreach ($scenarios as $scenario) { /** @var $scenario \Magento\TestFramework\Performance\Scenario */ $this->_application->applyFixtures($scenario->getFixtures()); $this->_notifyScenarioRun($scenario); /* warm up cache, if any */ $settings = $scenario->getSettings(); if (empty($settings[self::SETTING_SKIP_WARM_UP])) { try { $scenarioWarmUp = new \Magento\TestFramework\Performance\Scenario($scenario->getTitle(), $scenario->getFile(), $this->_warmUpArguments + $scenario->getArguments(), $scenario->getSettings(), $scenario->getFixtures()); $this->_scenarioHandler->run($scenarioWarmUp); } catch (\Magento\TestFramework\Performance\Scenario\FailureException $scenarioFailure) { // do not notify about failed warm up } } /* full run with reports recording */ $reportFile = $this->_getScenarioReportFile($scenario); try { $this->_scenarioHandler->run($scenario, $reportFile); } catch (\Magento\TestFramework\Performance\Scenario\FailureException $scenarioFailure) { $this->_notifyScenarioFailure($scenarioFailure); } } }