/**
* Calculate the progress for all of the images in a given directory
*/
function GetVisualProgress($testPath, $run, $cached, $options = null, $end = null, $startOffset = null)
{
// TODO: in the long run this function might get redundant as the version below is more flexible
$frames = null;
$testPath = $testPath[0] == '.' || $testPath[0] == "/" ? $testPath : "./{$testPath}";
$localPaths = new TestPaths($testPath, $run, $cached);
$testInfo = GetTestInfo($testPath);
$completed = IsTestRunComplete($run, $testInfo);
return GetVisualProgressForStep($localPaths, $completed, $options, $end, $startOffset);
}
/**
* @param TestPaths $localPaths Paths for this run/step to get the CPU time for
* @param int $endTime End time to consider (optional, will be retrieved from requests otherwise)
* @return array
*/
function GetDevToolsCPUTimeForStep($localPaths, $endTime = 0)
{
if (!$endTime) {
require_once __DIR__ . '/page_data.inc';
$runCompleted = IsTestRunComplete($localPaths->getRunNumber(), $testInfo);
$pageData = loadPageStepData($localPaths, $runCompleted);
if (isset($pageData) && is_array($pageData) && isset($pageData['fullyLoaded'])) {
$endTime = $pageData['fullyLoaded'];
}
}
$times = null;
$ver = 3;
$cacheFile = $localPaths->devtoolsCPUTimeCacheFile($ver);
if (gz_is_file($cacheFile)) {
$cache = json_decode(gz_file_get_contents($cacheFile), true);
}
if (isset($cache) && is_array($cache) && isset($cache[$endTime])) {
$times = $cache[$endTime];
} else {
$cpu = DevToolsGetCPUSlicesForStep($localPaths);
if (isset($cpu) && is_array($cpu) && isset($cpu['main_thread']) && isset($cpu['slices'][$cpu['main_thread']]) && isset($cpu['slice_usecs'])) {
$busy = 0;
$times = array();
if (!$endTime && isset($cpu['total_usecs'])) {
$endTime = $cpu['total_usecs'] / 1000;
}
foreach ($cpu['slices'][$cpu['main_thread']] as $name => $slices) {
$last_slice = min(intval(ceil($endTime * 1000 / $cpu['slice_usecs'])), count($slices));
$times[$name] = 0;
for ($i = 0; $i < $last_slice; $i++) {
$times[$name] += $slices[$i] / 1000.0;
}
$busy += $times[$name];
$times[$name] = intval(round($times[$name]));
}
$times['Idle'] = max($endTime - intval(round($busy)), 0);
}
// Cache the result
if (!isset($cache) || !is_array($cache)) {
$cache = array();
}
$cache[$endTime] = $times;
gz_file_put_contents($cacheFile, json_encode($cache));
}
return $times;
}
/**
* Calculate the progress for all of the images in a given directory
*/
function GetVisualProgress($testPath, $run, $cached, $options = null, $end = null, $startOffset = null)
{
$frames = null;
if (substr($testPath, 0, 1) !== '.') {
$testPath = './' . $testPath;
}
$testInfo = GetTestInfo($testPath);
$completed = IsTestRunComplete($run, $testInfo);
$video_directory = "{$testPath}/video_{$run}";
if ($cached) {
$video_directory .= '_cached';
}
$cache_file = "{$testPath}/{$run}.{$cached}.visual.dat";
if (!isset($startOffset)) {
$startOffset = 0;
}
$visual_data_file = "{$testPath}/llab_{$run}.{$cached}.visual.dat";
if (gz_is_file($visual_data_file)) {
$visual_data = json_decode(gz_file_get_contents($visual_data_file), true);
// see if we are processing an externally-uploaded visual data file
if (isset($visual_data['timespans']['page_load']['startOffset'])) {
$startOffset += $visual_data['timespans']['page_load']['startOffset'];
}
}
$dirty = false;
$current_version = VIDEO_CODE_VERSION;
if (isset($end)) {
if (is_numeric($end)) {
$end = (int) ($end * 1000);
} else {
unset($end);
}
}
if (!isset($end) && !isset($options) && gz_is_file($cache_file)) {
$frames = json_decode(gz_file_get_contents($cache_file), true);
if (!array_key_exists('frames', $frames) || !array_key_exists('version', $frames)) {
unset($frames);
} elseif (array_key_exists('version', $frames) && $frames['version'] !== $current_version) {
unset($frames);
}
}
$base_path = substr($video_directory, 1);
if ((!isset($frames) || !count($frames)) && (is_dir($video_directory) || gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json"))) {
$frames = array('version' => $current_version);
$frames['frames'] = array();
$dirty = true;
if (is_dir($video_directory)) {
$files = scandir($video_directory);
$last_file = null;
$first_file = null;
$previous_file = null;
foreach ($files as $file) {
if (strpos($file, 'frame_') !== false && strpos($file, '.hist') === false) {
$parts = explode('_', $file);
if (count($parts) >= 2) {
$time = (int) $parts[1] * 100 - $startOffset;
if ($time >= 0 && (!isset($end) || $time <= $end)) {
if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) {
$frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file);
$first_file = $previous_file;
} elseif (!isset($first_file)) {
$first_file = $file;
}
$last_file = $file;
$frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file);
}
$previous_file = $file;
}
} elseif (strpos($file, 'ms_') !== false && strpos($file, '.hist') === false) {
$parts = explode('_', $file);
if (count($parts) >= 2) {
$time = intval($parts[1]) - $startOffset;
if ($time >= 0 && (!isset($end) || $time <= $end)) {
if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) {
$frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file);
$first_file = $previous_file;
} elseif (!isset($first_file)) {
$first_file = $file;
}
$last_file = $file;
$frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file);
}
$previous_file = $file;
}
}
}
if (count($frames['frames']) == 1) {
foreach ($frames['frames'] as $time => &$frame) {
$frame['progress'] = 100;
$frames['complete'] = $time;
}
} elseif (isset($first_file) && strlen($first_file) && isset($last_file) && strlen($last_file) && count($frames['frames'])) {
$histograms = null;
if (gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json")) {
$histograms = json_decode(gz_file_get_contents("{$testPath}/{$run}.{$cached}.histograms.json"), true);
}
$start_histogram = GetImageHistogram("{$video_directory}/{$first_file}", $options, $histograms);
$final_histogram = GetImageHistogram("{$video_directory}/{$last_file}", $options, $histograms);
foreach ($frames['frames'] as $time => &$frame) {
$histogram = GetImageHistogram("{$video_directory}/{$frame['file']}", $options, $histograms);
$frame['progress'] = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5);
if ($frame['progress'] == 100 && !array_key_exists('complete', $frames)) {
$frames['complete'] = $time;
}
}
}
} elseif (gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json")) {
$raw = json_decode(gz_file_get_contents("{$testPath}/{$run}.{$cached}.histograms.json"), true);
$histograms = array();
foreach ($raw as $h) {
if (isset($h['time']) && isset($h['histogram'])) {
$histograms[$h['time']] = $h['histogram'];
}
}
ksort($histograms, SORT_NUMERIC);
$final_histogram = end($histograms);
$start_histogram = reset($histograms);
foreach ($histograms as $time => $histogram) {
$frames['frames'][$time] = array();
$progress = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5);
$frames['frames'][$time]['progress'] = $progress;
if ($progress == 100 && !isset($frames['complete'])) {
$frames['complete'] = $time;
}
}
}
}
if (isset($frames) && !array_key_exists('SpeedIndex', $frames)) {
$dirty = true;
$frames['SpeedIndex'] = CalculateSpeedIndex($frames);
}
if (isset($frames)) {
$frames['visualComplete'] = 0;
foreach ($frames['frames'] as $time => &$frame) {
if ($frame['progress'] > 0 && !array_key_exists('startRender', $frames)) {
$frames['startRender'] = $time;
}
if (!$frames['visualComplete'] && $frame['progress'] == 100) {
$frames['visualComplete'] = $time;
}
// fix up the frame paths in case we have a cached version referencing a relay path
if (isset($frame['path'])) {
$frame['path'] = $base_path . '/' . basename($frame['path']);
}
}
}
if ($completed && !isset($end) && !isset($options) && $dirty && isset($frames) && count($frames)) {
gz_file_put_contents($cache_file, json_encode($frames));
}
return $frames;
}
/**
* @param int $run The run to check if complete
* @return bool True if the test run is complete, false otherwise
*/
public function isRunComplete($run)
{
// TODO: move implementation here
return $run <= $this->getRuns() && IsTestRunComplete($run, $this->rawData['testinfo']);
}
/**
* Calculate the progress for all of the images in a given directory
*/
function GetVisualProgress($testPath, $run, $cached, $options = null, $end = null, $startOffset = null)
{
$frames = null;
if (substr($testPath, 0, 1) !== '.') {
$testPath = './' . $testPath;
}
$testInfo = GetTestInfo($testPath);
$completed = IsTestRunComplete($run, $testInfo);
$video_directory = "{$testPath}/video_{$run}";
if ($cached) {
$video_directory .= '_cached';
}
$cache_file = "{$testPath}/{$run}.{$cached}.visual.dat";
if (!isset($startOffset)) {
$startOffset = 0;
}
$dirty = false;
$current_version = VIDEO_CODE_VERSION;
if (isset($end)) {
if (is_numeric($end)) {
$end = (int) ($end * 1000);
} else {
unset($end);
}
}
if (!isset($end) && !isset($options) && gz_is_file($cache_file)) {
$frames = json_decode(gz_file_get_contents($cache_file), true);
if (!array_key_exists('frames', $frames) || !array_key_exists('version', $frames)) {
unset($frames);
} elseif (array_key_exists('version', $frames) && $frames['version'] !== $current_version) {
unset($frames);
}
}
if ((!isset($frames) || !count($frames)) && is_dir($video_directory)) {
$frames = array('version' => $current_version);
$frames['frames'] = array();
$dirty = true;
$base_path = substr($video_directory, 1);
$files = scandir($video_directory);
$last_file = null;
$first_file = null;
$previous_file = null;
foreach ($files as $file) {
if (strpos($file, 'frame_') !== false && strpos($file, '.hist') === false) {
$parts = explode('_', $file);
if (count($parts) >= 2) {
$time = (int) $parts[1] * 100 - $startOffset;
if ($time >= 0 && (!isset($end) || $time <= $end)) {
if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) {
$frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file);
$first_file = $previous_file;
} elseif (!isset($first_file)) {
$first_file = $file;
}
$last_file = $file;
$frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file);
}
$previous_file = $file;
}
} elseif (strpos($file, 'ms_') !== false && strpos($file, '.hist') === false) {
$parts = explode('_', $file);
if (count($parts) >= 2) {
$time = intval($parts[1]) - $startOffset;
if ($time >= 0 && (!isset($end) || $time <= $end)) {
if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) {
$frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file);
$first_file = $previous_file;
} elseif (!isset($first_file)) {
$first_file = $file;
}
$last_file = $file;
$frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file);
}
$previous_file = $file;
}
}
}
if (count($frames['frames']) == 1) {
foreach ($frames['frames'] as $time => &$frame) {
$frame['progress'] = 100;
$frames['complete'] = $time;
}
} elseif (isset($first_file) && strlen($first_file) && isset($last_file) && strlen($last_file) && count($frames['frames'])) {
$start_histogram = GetImageHistogram("{$video_directory}/{$first_file}", $options);
$final_histogram = GetImageHistogram("{$video_directory}/{$last_file}", $options);
foreach ($frames['frames'] as $time => &$frame) {
$histogram = GetImageHistogram("{$video_directory}/{$frame['file']}", $options);
$frame['progress'] = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5);
if ($frame['progress'] == 100 && !array_key_exists('complete', $frames)) {
$frames['complete'] = $time;
}
}
}
}
if (isset($frames) && !array_key_exists('SpeedIndex', $frames)) {
$dirty = true;
$frames['SpeedIndex'] = CalculateSpeedIndex($frames);
}
if (isset($frames)) {
$frames['visualComplete'] = 0;
foreach ($frames['frames'] as $time => &$frame) {
if ($frame['progress'] > 0 && !array_key_exists('startRender', $frames)) {
$frames['startRender'] = $time;
}
if ($frame['progress'] == 100) {
$frames['visualComplete'] = $time;
break;
}
}
}
$devTools = GetDevToolsProgress($testPath, $run, $cached);
if (isset($devTools)) {
if (!isset($frames)) {
$frames = array();
}
$frames['DevTools'] = $devTools;
}
if ($completed && !isset($end) && !isset($options) && $dirty && isset($frames) && count($frames)) {
gz_file_put_contents($cache_file, json_encode($frames));
}
return $frames;
}
/**
* Calculate the visual progress and speed index from the dev tools timeline trace
*
* @param mixed $testPath
* @param mixed $run
* @param mixed $cached
*/
function GetDevToolsProgress($testPath, $run, $cached)
{
$progress = GetCachedDevToolsProgress($testPath, $run, $cached);
if (!isset($progress) || !is_array($progress)) {
$testInfo = GetTestInfo($testPath);
$completed = IsTestRunComplete($run, $testInfo);
$startOffset = null;
if (GetTimeline($testPath, $run, $cached, $timeline, $startOffset)) {
$cachedText = '';
if ($cached) {
$cachedText = '_Cached';
}
$console_log_file = "{$testPath}/{$run}{$cachedText}_console_log.json";
$console_log = array();
$progress = array();
$startTime = 0;
$fullScreen = 0;
$regions = array();
$viewport = null;
if (DevToolsHasLayout($timeline, $viewport)) {
$didLayout = false;
$didReceiveResponse = false;
} else {
$didLayout = true;
$didReceiveResponse = true;
}
$startTimes = array();
$progress['processing'] = array();
foreach ($timeline as &$entry) {
if (array_key_exists('method', $entry)) {
if (array_key_exists('params', $entry) && !array_key_exists($entry['method'], $startTimes)) {
if (array_key_exists('timestamp', $entry['params'])) {
$startTimes[$entry['method']] = $entry['params']['timestamp'];
} elseif (array_key_exists('record', $entry['params']) && array_key_exists('startTime', $entry['params']['record'])) {
$startTimes[$entry['method']] = $entry['params']['record']['startTime'];
}
}
} elseif (array_key_exists('timestamp', $entry) && !array_key_exists('timestamp', $startTimes)) {
$startTimes['timestamp'] = $entry['timestamp'];
}
$frame = '0';
ProcessPaintEntry($entry, $fullScreen, $regions, $frame, $didLayout, $didReceiveResponse, $viewport);
if (!isset($entry['params']['record']['thread']) || $entry['params']['record']['thread'] == 0) {
GetTimelineProcessingTimes($entry, $progress['processing'], $processing_start, $processing_end);
}
if (DevToolsMatchEvent('Console.messageAdded', $entry) && array_key_exists('message', $entry['params']) && is_array($entry['params']['message'])) {
$console_log[] = $entry['params']['message'];
}
}
if (!gz_is_file($console_log_file)) {
gz_file_put_contents($console_log_file, json_encode($console_log));
}
if (count($progress['processing'])) {
$proc_total = 0.0;
foreach ($progress['processing'] as $type => &$procTime) {
$proc_total += $procTime;
$procTime = intval(round($procTime));
}
$progress['processing']['Idle'] = 0;
if (isset($processing_start) && isset($processing_end) && $processing_end > $processing_start) {
$proc_elapsed = $processing_end - $processing_start;
if ($proc_elapsed > $proc_total) {
$progress['processing']['Idle'] = intval(round($proc_elapsed - $proc_total));
}
}
} else {
unset($progress['processing']);
}
foreach ($startTimes as $time) {
if (!$startTime || $time < $startTime) {
$startTime = $time;
}
}
$regionCount = count($regions);
if ($regionCount) {
$paintEvents = array();
$total = 0.0;
foreach ($regions as $name => &$region) {
$area = $region['width'] * $region['height'];
$updateCount = floatval(count($region['times']));
$incrementalImpact = floatval($area) / $updateCount;
// only count full screen paints for half their value
if ($area == $fullScreen) {
$incrementalImpact /= 2;
}
foreach ($region['times'] as $time) {
$total += $incrementalImpact;
$elapsed = (int) ($time - $startTime);
if (!array_key_exists($elapsed, $paintEvents)) {
$paintEvents[$elapsed] = $incrementalImpact;
} else {
$paintEvents[$elapsed] += $incrementalImpact;
}
}
}
if (count($paintEvents)) {
ksort($paintEvents, SORT_NUMERIC);
$current = 0.0;
$lastTime = 0.0;
$lastProgress = 0.0;
$progress['SpeedIndex'] = 0.0;
$progress['VisuallyComplete'] = 0;
$progress['StartRender'] = 0;
$progress['VisualProgress'] = array();
foreach ($paintEvents as $time => $increment) {
$current += $increment;
$currentProgress = floatval(floatval($current) / floatval($total));
$currentProgress = floatval(round($currentProgress * 100) / 100.0);
$elapsed = $time - $lastTime;
$siIncrement = floatval($elapsed) * (1.0 - $lastProgress);
$progress['SpeedIndex'] += $siIncrement;
$progress['VisualProgress'][$time] = $currentProgress;
$progress['VisuallyComplete'] = $time;
if (!$progress['StartRender']) {
$progress['StartRender'] = $time;
}
$lastProgress = $currentProgress;
$lastTime = $time;
if ($currentProgress >= 1.0) {
break;
}
}
}
}
if ($completed && isset($progress) && is_array($progress)) {
SavedCachedDevToolsProgress($testPath, $run, $cached, $progress);
}
}
}
return $progress;
}
