function ProcessFile($file) { global $benchmark; echo "Reprocessing {$file}\n"; logMsg("Reprocessing {$file}", "./log/reprocess-{$benchmark}.log", true); $tests = array(); $entries = json_decode(gz_file_get_contents($file), true); if (isset($entries) && is_array($entries) && count($entries)) { echo 'Loaded ' . count($entries) . " results\n"; foreach ($entries as $entry) { if (!array_key_exists($entry['id'], $tests)) { $tests[$entry['id']] = array('url' => $entry['url'], 'label' => $entry['label'], 'location' => $entry['location'], 'config' => $entry['config'], 'id' => $entry['id']); } } unset($entries); } if (count($tests)) { $results = array(); foreach ($tests as &$test) { CollectTestResult($test, $results); } echo 'Writing ' . count($results) . " results\n"; rename($file, "{$file}.reprocessed"); gz_file_put_contents(str_replace('.json.gz', '.json', $file), json_encode($results)); } return $ok; }
/** * For each IP/Installer pair, keep track of the last 4 checks and if they * were within the last hour fail the request. * * @param mixed $installer */ function CheckIp($installer) { $ok = true; $ip = $_SERVER["REMOTE_ADDR"]; if (isset($ip) && strlen($ip)) { $lock = Lock("Installers", true, 5); if ($lock) { $now = time(); $file = "./tmp/installers.dat"; if (gz_is_file($file)) { $history = json_decode(gz_file_get_contents($file), true); } if (!isset($history) || !is_array($history)) { $history = array(); } if (isset($history[$ip])) { if (isset($history[$ip][$installer])) { $history[$ip][$installer][] = $now; if (count($history[$ip][$installer]) > 10) { array_shift($history[$ip][$installer]); } if (isset($history[$ip]["last-{$installer}"]) && $now - $history[$ip]["last-{$installer}"] < 3600) { $count = 0; foreach ($history[$ip][$installer] as $time) { if ($now - $time < 3600) { $count++; } } if ($count > 4) { $ok = false; } } } else { $history[$ip][$installer] = array($now); } } else { $history[$ip] = array($installer => array($now)); } $history[$ip]['last'] = $now; if ($ok) { $history[$ip]["last-{$installer}"] = $now; } // prune any agents that haven't connected in 7 days foreach ($history as $agent => $info) { if ($now - $info['last'] > 604800) { unset($history[$agent]); } } gz_file_put_contents($file, json_encode($history)); Unlock($lock); } } return $ok; }
function GetLastFrameHistogram($test, $run) { $histogram = null; $testPath = GetTestPath($test); $videoPath = "./{$testPath}/video_{$run}"; $files = glob("{$videoPath}/*.jpg"); if ($files) { rsort($files); $lastFrame = $files[0]; if (gz_is_file("{$testPath}/{$run}.0.histograms.json")) { $histograms = json_decode(gz_file_get_contents("{$testPath}/{$run}.0.histograms.json"), true); } $histogram = GetImageHistogram($lastFrame, null, $histograms); } return $histogram; }
function ResubmitTest($id) { echo "{$id} - Resubmitting..."; $testPath = './' . GetTestPath($id); if (gz_is_file("{$testPath}/testinfo.json")) { $test = json_decode(gz_file_get_contents("{$testPath}/testinfo.json"), true); if (array_key_exists('job_file', $test) && array_key_exists('location', $test) && is_file("{$testPath}/test.job")) { if ($lock = LockLocation($test['location'])) { if (copy("{$testPath}/test.job", $test['job_file'])) { $files = scandir($testPath); foreach ($files as $file) { if ($file != '.' && $file != '..' && strncasecmp($file, 'test', 4)) { if (is_file("{$testPath}/{$file}")) { unlink("{$testPath}/{$file}"); } elseif (is_dir("{$testPath}/{$file}")) { delTree("{$testPath}/{$file}"); } } } AddJobFile($test['workdir'], $test['job'], $test['priority'], false); $test['started'] = time(); unset($test['completeTime']); gz_file_put_contents("{$testPath}/testinfo.json", json_encode($test)); echo "OK"; } else { echo "Failed to copy job file"; } UnlockLocation($lock); } else { echo "Failed to lock location"; } } else { echo "Invalid test"; } } else { echo "Test not found"; } echo "\n"; }
public function testPostProcessRun() { $firstStepPaths = new TestPaths($this->resultDir, 1, true, 1); $secondStepPaths = new TestPaths($this->resultDir, 1, true, 2); $csiCacheFile = $firstStepPaths->csiCacheFile(CSI_CACHE_VERSION) . ".gz"; $breakdownCacheFile = $firstStepPaths->breakdownCacheFile(BREAKDOWN_CACHE_VERSION) . ".gz"; $this->assertEquals($csiCacheFile, $secondStepPaths->csiCacheFile(CSI_CACHE_VERSION) . ".gz"); $this->assertEquals($breakdownCacheFile, $secondStepPaths->breakdownCacheFile(BREAKDOWN_CACHE_VERSION) . ".gz"); $this->assertNotEquals($firstStepPaths->cacheKey(), $secondStepPaths->cacheKey()); $this->assertFileNotExists($csiCacheFile); $this->assertFileNotExists($breakdownCacheFile); $resultProcessing = new ResultProcessing($this->resultDir, $this->testId, 1, true); $error = $resultProcessing->postProcessRun(); $this->assertNull($error); $this->assertFileExists($csiCacheFile); $cache = json_decode(gz_file_get_contents($csiCacheFile), true); $this->assertNotEmpty(@$cache[$firstStepPaths->cacheKey()], "CSI Cache for step 1 is empty"); // actually, the second step doesn't contain CSI info. But it should be in the cache anyway $this->assertArrayHasKey($secondStepPaths->cacheKey(), $cache, "CSI Cache for step 2 is empty"); $this->assertFileExists($breakdownCacheFile); $cache = json_decode(gz_file_get_contents($breakdownCacheFile), true); $this->assertNotEmpty(@$cache[$firstStepPaths->cacheKey()], "Breakdown Cache for step 1 is empty"); $this->assertNotEmpty(@$cache[$secondStepPaths->cacheKey()], "Breakdown Cache for step 2 is empty"); }
/** * Remove sensitive data fields from HTTP headers (cookies and HTTP Auth) * */ function RemoveSensitiveHeaders($file) { $patterns = array('/(cookie:[ ]*)([^\\r\\n]*)/i', '/(authenticate:[ ]*)([^\\r\\n]*)/i'); $data = gz_file_get_contents($file); $data = preg_replace($patterns, '\\1XXXXXX', $data); gz_file_put_contents($file, $data); }
<?php include 'common.inc'; $ok = false; if (isset($_GET['file']) && strlen($_GET['file'])) { $file = $_GET['file']; if (strpos($file, '/') === false && strpos($file, '\\') === false && strpos($file, '..') === false) { $data = gz_file_get_contents("{$testPath}/{$_GET['file']}"); if ($data !== false) { $ok = true; echo $data; } } } if (!$ok) { header("HTTP/1.0 404 Not Found"); }
$users = array(); } if (isset($users[$user['email']]['id'])) { $user['id'] = $users[$user['email']]['id']; } $users[$user['email']] = $user; gz_file_put_contents('./dat/users.dat', json_encode($users)); // se if the user that logged in was an administrator $admin_users = GetSetting('admin_users'); if ($admin_users) { $admins = explode(',', $admin_users); foreach ($admins as $admin) { if (stripos(strtolower($user['email']), strtolower(trim($admin))) !== false) { $session = sha1(json_encode($token_data) . time()); setcookie("asid", $session, time() + 60 * 60 * 24 * 7 * 2, "/"); $sessions = json_decode(gz_file_get_contents('./dat/admin_sessions.dat'), true); if (!isset($sessions) || !is_array($sessions)) { $sessions = array(); } $sessions[$session] = $user; gz_file_put_contents('./dat/admin_sessions.dat', json_encode($sessions)); break; } } } Unlock($lock); } setcookie("google_id", $user['id'], time() + 60 * 60 * 24 * 7 * 2, "/"); setcookie("google_email", $user['email'], time() + 60 * 60 * 24 * 7 * 2, "/"); $redirect = isset($_COOKIE['page_before_google_oauth']) ? $_COOKIE['page_before_google_oauth'] : "{$protocol}://{$host}/"; header('Location: ' . $redirect);
/** * Create the various aggregations for the given data chunk * * @param mixed $info * @param mixed $data * @param mixed $benchmark */ function CreateAggregates(&$info, &$data, $benchmark, $run_time, $options) { foreach ($info['metrics'] as $metric) { $metric_file = "./results/benchmarks/{$benchmark}/aggregate/{$metric}.json"; if (gz_is_file($metric_file)) { $agg_data = json_decode(gz_file_get_contents($metric_file), true); } else { $agg_data = array(); } AggregateMetric($metric, $info, $data, $run_time, $agg_data, $options); gz_file_put_contents($metric_file, @json_encode($agg_data)); unset($agg_data); if (array_key_exists('labels', $info) && count($info['labels']) <= 20) { $metric_file = "./results/benchmarks/{$benchmark}/aggregate/{$metric}.labels.json"; if (gz_is_file($metric_file)) { $agg_data = json_decode(gz_file_get_contents($metric_file), true); } else { $agg_data = array(); } AggregateMetricByLabel($metric, $info, $data, $run_time, $agg_data, $options); gz_file_put_contents($metric_file, json_encode($agg_data)); unset($agg_data); } } }
header("Content-disposition: attachment; filename={$filename}"); header("Content-type: text/csv"); header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); if ($test['test']['batch']) { $tests = null; if (gz_is_file("{$testPath}/tests.json")) { $legacyData = json_decode(gz_file_get_contents("{$testPath}/tests.json"), true); $tests = array(); $tests['variations'] = array(); $tests['urls'] = array(); foreach ($legacyData as &$legacyTest) { $tests['urls'][] = array('u' => $legacyTest['url'], 'id' => $legacyTest['id']); } } elseif (gz_is_file("{$testPath}/bulk.json")) { $tests = json_decode(gz_file_get_contents("{$testPath}/bulk.json"), true); } if (isset($tests)) { foreach ($tests['urls'] as &$testData) { RestoreTest($testData['id']); $path = './' . GetTestPath($testData['id']); if (!isset($hasCSV)) { $files = glob("{$path}/*{$fileType}"); if ($files && is_array($files) && count($files)) { $hasCSV = true; } else { $hasCSV = false; } } $label = $testData['l']; if (!strlen($label)) {
require_once 'waterfall.inc'; $page_keywords = array('Performance Test', 'Details', 'Webpagetest', 'Website Speed Test', 'Page Speed'); $page_description = "Website performance test comparison"; $fvonly = false; $metrics = array('docTime' => 'Load Time (onload)', 'fullyLoaded' => 'Load Time (Fully Loaded)', 'TTFB' => 'First Byte Time', 'render' => 'Time to First Paint', 'requestsDoc' => 'Requests', 'bytesInDoc' => 'Bytes In'); // load the data for each of the tests $tests = array(); $fv_max_time = 0; if (array_key_exists('tests', $_REQUEST)) { $parts = explode(',', $_REQUEST['tests']); foreach ($parts as $fragment) { $test = array(); $components = explode('-', $fragment); $test['id'] = trim($components[0]); $test['path'] = './' . GetTestPath($test['id']); $test['info'] = json_decode(gz_file_get_contents("{$test['path']}/testinfo.json"), true); $test['page_data'] = loadAllPageData($test['path']); $test['fv_run'] = GetMedianRun($test['page_data'], 0, $median_metric); if ($test['fv_run']) { $test['fv'] = $test['page_data'][$test['fv_run']][0]; $docTime = $test['fv']['docTime'] / 1000; if ($docTime > $fv_max_time) { $fv_max_time = $docTime; } } $test['rv_run'] = GetMedianRun($test['page_data'], 1, $median_metric); if ($test['rv_run']) { $test['rv'] = $test['page_data'][$test['rv_run']][0]; } else { $fvonly = true; }
/** * Calculate the progress for all of the images in a given directory */ function GetVisualProgress($testPath, $run, $cached, $options = null, $end = null, $startOffset = null) { $frames = null; if (substr($testPath, 0, 1) !== '.') { $testPath = './' . $testPath; } $testInfo = GetTestInfo($testPath); $completed = IsTestRunComplete($run, $testInfo); $video_directory = "{$testPath}/video_{$run}"; if ($cached) { $video_directory .= '_cached'; } $cache_file = "{$testPath}/{$run}.{$cached}.visual.dat"; if (!isset($startOffset)) { $startOffset = 0; } $dirty = false; $current_version = VIDEO_CODE_VERSION; if (isset($end)) { if (is_numeric($end)) { $end = (int) ($end * 1000); } else { unset($end); } } if (!isset($end) && !isset($options) && gz_is_file($cache_file)) { $frames = json_decode(gz_file_get_contents($cache_file), true); if (!array_key_exists('frames', $frames) || !array_key_exists('version', $frames)) { unset($frames); } elseif (array_key_exists('version', $frames) && $frames['version'] !== $current_version) { unset($frames); } } if ((!isset($frames) || !count($frames)) && is_dir($video_directory)) { $frames = array('version' => $current_version); $frames['frames'] = array(); $dirty = true; $base_path = substr($video_directory, 1); $files = scandir($video_directory); $last_file = null; $first_file = null; $previous_file = null; foreach ($files as $file) { if (strpos($file, 'frame_') !== false && strpos($file, '.hist') === false) { $parts = explode('_', $file); if (count($parts) >= 2) { $time = (int) $parts[1] * 100 - $startOffset; if ($time >= 0 && (!isset($end) || $time <= $end)) { if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) { $frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file); $first_file = $previous_file; } elseif (!isset($first_file)) { $first_file = $file; } $last_file = $file; $frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file); } $previous_file = $file; } } elseif (strpos($file, 'ms_') !== false && strpos($file, '.hist') === false) { $parts = explode('_', $file); if (count($parts) >= 2) { $time = intval($parts[1]) - $startOffset; if ($time >= 0 && (!isset($end) || $time <= $end)) { if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) { $frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file); $first_file = $previous_file; } elseif (!isset($first_file)) { $first_file = $file; } $last_file = $file; $frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file); } $previous_file = $file; } } } if (count($frames['frames']) == 1) { foreach ($frames['frames'] as $time => &$frame) { $frame['progress'] = 100; $frames['complete'] = $time; } } elseif (isset($first_file) && strlen($first_file) && isset($last_file) && strlen($last_file) && count($frames['frames'])) { $start_histogram = GetImageHistogram("{$video_directory}/{$first_file}", $options); $final_histogram = GetImageHistogram("{$video_directory}/{$last_file}", $options); foreach ($frames['frames'] as $time => &$frame) { $histogram = GetImageHistogram("{$video_directory}/{$frame['file']}", $options); $frame['progress'] = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5); if ($frame['progress'] == 100 && !array_key_exists('complete', $frames)) { $frames['complete'] = $time; } } } } if (isset($frames) && !array_key_exists('SpeedIndex', $frames)) { $dirty = true; $frames['SpeedIndex'] = CalculateSpeedIndex($frames); } if (isset($frames)) { $frames['visualComplete'] = 0; foreach ($frames['frames'] as $time => &$frame) { if ($frame['progress'] > 0 && !array_key_exists('startRender', $frames)) { $frames['startRender'] = $time; } if ($frame['progress'] == 100) { $frames['visualComplete'] = $time; break; } } } $devTools = GetDevToolsProgress($testPath, $run, $cached); if (isset($devTools)) { if (!isset($frames)) { $frames = array(); } $frames['DevTools'] = $devTools; } if ($completed && !isset($end) && !isset($options) && $dirty && isset($frames) && count($frames)) { gz_file_put_contents($cache_file, json_encode($frames)); } return $frames; }
/** * Load the test data and keep just the median run for each config */ function LoadMedianData($benchmark, $test_time) { global $median_data; global $raw_data; if (!isset($median_data)) { // see if we have a custom metric to use to calculate the median for the given benchmark $info = GetBenchmarkInfo($benchmark); $median_metric = 'docTime'; if (isset($info) && is_array($info) && array_key_exists('options', $info) && array_key_exists('median_run', $info['options'])) { $median_metric = $info['options']['median_run']; } $date = gmdate('Ymd_Hi', $test_time); $data_file = "./results/benchmarks/{$benchmark}/data/{$date}.json"; $key = "{$benchmark}-{$date}"; if (gz_is_file($data_file)) { if (!array_key_exists($key, $raw_data)) { $raw_data[$key] = json_decode(gz_file_get_contents($data_file), true); usort($raw_data[$key], 'RawDataCompare'); } if (count($raw_data[$key])) { $tests = array(); // group the results by test ID foreach ($raw_data[$key] as &$row) { if (array_key_exists($median_metric, $row) && ($row['result'] == 0 || $row['result'] == 99999)) { $id = $row['id']; $cached = $row['cached']; $key = "{$id}-{$cached}"; if (!array_key_exists($key, $tests)) { $tests[$key] = array(); } $tests[$key][] = $row; } } // extract just the median runs $median_data = array(); foreach ($tests as &$test) { $times = array(); foreach ($test as $row) { $times[] = $row[$median_metric]; } $median_run_index = 0; $count = count($times); if ($count > 1) { asort($times); $medianIndex = (int) floor(((double) $count + 1.0) / 2.0); $current = 0; foreach ($times as $index => $time) { $current++; if ($current == $medianIndex) { $median_run_index = $index; break; } } } $row = $test[$median_run_index]; if (array_key_exists('cached', $row) && array_key_exists('url', $row) && array_key_exists('config', $row) && array_key_exists('location', $row)) { $url = $row['url']; if (array_key_exists('label', $row) && strlen($row['label'])) { $url = $row['label']; } $config = $row['config']; $location = $row['location']; $cached = $row['cached']; if (isset($loc_aliases) && count($loc_aliases)) { foreach ($loc_aliases as $loc_name => &$aliases) { foreach ($aliases as $alias) { if ($location == $alias) { $location = $loc_name; break 2; } } } } if (!array_key_exists($config, $median_data)) { $median_data[$config] = array(); } if (!array_key_exists($location, $median_data[$config])) { $median_data[$config][$location] = array(); } if (!array_key_exists($cached, $median_data[$config][$location])) { $median_data[$config][$location][$cached] = array(); } $median_data[$config][$location][$cached][$url] = $row; } } } } } }
/** * If we have a timeline, figure out what each thread was doing at each point in time. * Basically CPU utilization from the timeline. * * returns an array of threads with each thread being an array of slices (one for * each time period). Each slice is an array of events and the fraction of that * slice that they consumed (with a total maximum of 1 for any slice). */ function DevToolsGetCPUSlices($testPath, $run, $cached) { $count = 0; $slices = null; $devTools = array(); $startOffset = null; $ver = 1; $cacheFile = "{$testPath}/{$run}.{$cached}.devToolsCPUSlices.{$ver}"; if (gz_is_file($cacheFile)) { $slices = json_decode(gz_file_get_contents($cacheFile), true); } if (!isset($slices)) { GetTimeline($testPath, $run, $cached, $devTools, $startOffset); if (isset($devTools) && is_array($devTools) && count($devTools)) { // Do a first pass to get the start and end times as well as the number of threads $threads = array(0 => true); $startTime = 0; $endTime = 0; foreach ($devTools as &$entry) { if (isset($entry['method']) && $entry['method'] == 'Timeline.eventRecorded' && isset($entry['params']['record'])) { $start = DevToolsEventTime($entry); if ($start && (!$startTime || $start < $startTime)) { $startTime = $start; } $end = DevToolsEventEndTime($entry); if ($end && (!$endTime || $end > $endTime)) { $endTime = $end; } $thread = isset($entry['params']['record']['thread']) ? $entry['params']['record']['thread'] : 0; $threads[$thread] = true; } } // create time slice arrays for each thread $slices = array(); foreach ($threads as $id => $bogus) { $slices[$id] = array(); } // create 1ms time slices for the full time if ($endTime > $startTime) { $startTime = floor($startTime); $endTime = ceil($endTime); for ($i = $startTime; $i <= $endTime; $i++) { $ms = intval($i - $startTime); foreach ($threads as $id => $bogus) { $slices[$id][$ms] = array(); } } // Go through each element and account for the time foreach ($devTools as &$entry) { if (isset($entry['method']) && $entry['method'] == 'Timeline.eventRecorded' && isset($entry['params']['record'])) { $count += DevToolsGetEventTimes($entry['params']['record'], $startTime, $slices); } } } } if ($count) { // remove any threads that didn't have actual slices populated $emptyThreads = array(); foreach ($slices as $thread => &$records) { $is_empty = true; foreach ($records as $ms => &$values) { if (count($values)) { $is_empty = false; break; } } if ($is_empty) { $emptyThreads[] = $thread; } } if (count($emptyThreads)) { foreach ($emptyThreads as $thread) { unset($slices[$thread]); } } gz_file_put_contents($cacheFile, json_encode($slices)); } else { $slices = null; } } return $slices; }
function GetDevToolsCPUTime($testPath, $run, $cached, $endTime = 0) { $times = null; $ver = 1; $ver = 2; $cacheFile = "{$testPath}/{$run}.{$cached}.devToolsCPUTime.{$ver}"; if (gz_is_file($cacheFile)) { $cache = json_decode(gz_file_get_contents($cacheFile), true); } // If an end time wasn't specified, figure out what the fully loaded time is if (!$endTime) { if (GetDevToolsRequests($testPath, $run, $cached, $requests, $pageData) && isset($pageData) && is_array($pageData) && isset($pageData['fullyLoaded'])) { $endTime = $pageData['fullyLoaded']; } } if (isset($cache[$endTime])) { $times = $cache[$endTime]; } else { $slices = DevToolsGetCPUSlices($testPath, $run, $cached); if (isset($slices) && is_array($slices) && isset($slices[0]) && is_array($slices[0]) && count($slices[0])) { $times = array('Idle' => 0.0); foreach ($slices[0] as $ms => $breakdown) { if (!$endTime || $ms < $endTime) { $idle = 1.0; if (isset($breakdown) && is_array($breakdown) && count($breakdown)) { foreach ($breakdown as $event => $ms_time) { if (!isset($times[$event])) { $times[$event] = 0; } $times[$event] += $ms_time; $idle -= $ms_time; } } $times['Idle'] += $idle; } } // round the times to the nearest millisecond $total = 0; foreach ($times as $event => &$val) { $val = round($val); if ($event !== 'Idle') { $total += $val; } } if ($endTime && $endTime > $total) { $times['Idle'] = $endTime - $total; } } $cache[$endTime] = $times; gz_file_put_contents($cacheFile, json_encode($cache)); } return $times; }
/** * Send back the data for a batch test (just the list of test ID's) * * @param mixed $id * @param mixed $testPath */ function BatchResult($id, $testPath) { header('Content-type: text/xml'); echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; echo "<response>"; if (strlen($_REQUEST['r'])) { echo "<requestId>{$_REQUEST['r']}</requestId>"; } $tests = null; if (gz_is_file("{$testPath}/bulk.json")) { $tests = json_decode(gz_file_get_contents("{$testPath}/bulk.json"), true); } elseif (gz_is_file("{$testPath}/tests.json")) { $legacyData = json_decode(gz_file_get_contents("{$testPath}/tests.json"), true); $tests = array(); $tests['variations'] = array(); $tests['urls'] = array(); foreach ($legacyData as &$legacyTest) { $tests['urls'][] = array('u' => $legacyTest['url'], 'id' => $legacyTest['id']); } } if (count($tests['urls'])) { echo "<statusCode>200</statusCode>"; echo "<statusText>Ok</statusText>"; if (strlen($_REQUEST['r'])) { echo "<requestId>{$_REQUEST['r']}</requestId>"; } $host = $_SERVER['HTTP_HOST']; $uri = rtrim(dirname($_SERVER['PHP_SELF']), '/\\'); echo "<data>"; foreach ($tests['urls'] as &$test) { echo "<test>"; echo "<testId>{$test['id']}</testId>"; echo "<testUrl>" . xml_entities($test['u']) . "</testUrl>"; echo "<xmlUrl>http://{$host}{$uri}/xmlResult/{$test['id']}/</xmlUrl>"; echo "<userUrl>http://{$host}{$uri}/result/{$test['id']}/</userUrl>"; echo "<summaryCSV>http://{$host}{$uri}/result/{$test['id']}/page_data.csv</summaryCSV>"; echo "<detailCSV>http://{$host}{$uri}/result/{$test['id']}/requests.csv</detailCSV>"; echo "</test>"; // go through all of the variations as well foreach ($test['v'] as $variationIndex => $variationId) { echo "<test>"; echo "<testId>{$variationId}</testId>"; echo "<testUrl>" . xml_entities(CreateUrlVariation($test['u'], $tests['variations'][$variationIndex]['q'])) . "</testUrl>"; echo "<xmlUrl>http://{$host}{$uri}/xmlResult/{$variationId}/</xmlUrl>"; echo "<userUrl>http://{$host}{$uri}/result/{$variationId}/</userUrl>"; echo "<summaryCSV>http://{$host}{$uri}/result/{$variationId}/page_data.csv</summaryCSV>"; echo "<detailCSV>http://{$host}{$uri}/result/{$variationId}/requests.csv</detailCSV>"; echo "</test>"; } } echo "</data>"; } else { echo "<statusCode>404</statusCode>"; echo "<statusText>Invalid Test ID: {$id}</statusText>"; echo "<data>"; echo "</data>"; } echo "</response>"; }
$date = gmdate("M j, Y", filemtime("./{$dir}/video.mp4")); } $title .= " - {$date}"; $labels = json_decode(file_get_contents("./{$dir}/labels.txt"), true); if (count($labels)) { $title .= ' : '; foreach ($labels as $index => $label) { if ($index > 0) { $title .= ", "; } $title .= $label; } } $location = null; if (gz_is_file("./{$dir}/testinfo.json")) { $tests = json_decode(gz_file_get_contents("./{$dir}/testinfo.json"), true); if (is_array($tests) && count($tests)) { foreach ($tests as &$test) { if (array_key_exists('location', $test)) { if (!isset($location)) { $location = $test['location']; } elseif ($location != $test['location']) { $location = ''; } } else { $location = ''; } } } } }
/** * @param TestPaths $localPaths Paths for this run/step to get the CPU time for * @param int $endTime End time to consider (optional, will be retrieved from requests otherwise) * @return array */ function GetDevToolsCPUTimeForStep($localPaths, $endTime = 0) { if (!$endTime) { require_once __DIR__ . '/page_data.inc'; $runCompleted = IsTestRunComplete($localPaths->getRunNumber(), $testInfo); $pageData = loadPageStepData($localPaths, $runCompleted); if (isset($pageData) && is_array($pageData) && isset($pageData['fullyLoaded'])) { $endTime = $pageData['fullyLoaded']; } } $times = null; $ver = 3; $cacheFile = $localPaths->devtoolsCPUTimeCacheFile($ver); if (gz_is_file($cacheFile)) { $cache = json_decode(gz_file_get_contents($cacheFile), true); } if (isset($cache) && is_array($cache) && isset($cache[$endTime])) { $times = $cache[$endTime]; } else { $cpu = DevToolsGetCPUSlicesForStep($localPaths); if (isset($cpu) && is_array($cpu) && isset($cpu['main_thread']) && isset($cpu['slices'][$cpu['main_thread']]) && isset($cpu['slice_usecs'])) { $busy = 0; $times = array(); if (!$endTime && isset($cpu['total_usecs'])) { $endTime = $cpu['total_usecs'] / 1000; } foreach ($cpu['slices'][$cpu['main_thread']] as $name => $slices) { $last_slice = min(intval(ceil($endTime * 1000 / $cpu['slice_usecs'])), count($slices)); $times[$name] = 0; for ($i = 0; $i < $last_slice; $i++) { $times[$name] += $slices[$i] / 1000.0; } $busy += $times[$name]; $times[$name] = intval(round($times[$name])); } $times['Idle'] = max($endTime - intval(round($busy)), 0); } // Cache the result if (!isset($cache) || !is_array($cache)) { $cache = array(); } $cache[$endTime] = $times; gz_file_put_contents($cacheFile, json_encode($cache)); } return $times; }
/** * Calculate the progress for all of the images in a given directory */ function GetVisualProgress($testPath, $run, $cached, $options = null, $end = null, $startOffset = null) { $frames = null; if (substr($testPath, 0, 1) !== '.') { $testPath = './' . $testPath; } $testInfo = GetTestInfo($testPath); $completed = IsTestRunComplete($run, $testInfo); $video_directory = "{$testPath}/video_{$run}"; if ($cached) { $video_directory .= '_cached'; } $cache_file = "{$testPath}/{$run}.{$cached}.visual.dat"; if (!isset($startOffset)) { $startOffset = 0; } $visual_data_file = "{$testPath}/llab_{$run}.{$cached}.visual.dat"; if (gz_is_file($visual_data_file)) { $visual_data = json_decode(gz_file_get_contents($visual_data_file), true); // see if we are processing an externally-uploaded visual data file if (isset($visual_data['timespans']['page_load']['startOffset'])) { $startOffset += $visual_data['timespans']['page_load']['startOffset']; } } $dirty = false; $current_version = VIDEO_CODE_VERSION; if (isset($end)) { if (is_numeric($end)) { $end = (int) ($end * 1000); } else { unset($end); } } if (!isset($end) && !isset($options) && gz_is_file($cache_file)) { $frames = json_decode(gz_file_get_contents($cache_file), true); if (!array_key_exists('frames', $frames) || !array_key_exists('version', $frames)) { unset($frames); } elseif (array_key_exists('version', $frames) && $frames['version'] !== $current_version) { unset($frames); } } $base_path = substr($video_directory, 1); if ((!isset($frames) || !count($frames)) && (is_dir($video_directory) || gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json"))) { $frames = array('version' => $current_version); $frames['frames'] = array(); $dirty = true; if (is_dir($video_directory)) { $files = scandir($video_directory); $last_file = null; $first_file = null; $previous_file = null; foreach ($files as $file) { if (strpos($file, 'frame_') !== false && strpos($file, '.hist') === false) { $parts = explode('_', $file); if (count($parts) >= 2) { $time = (int) $parts[1] * 100 - $startOffset; if ($time >= 0 && (!isset($end) || $time <= $end)) { if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) { $frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file); $first_file = $previous_file; } elseif (!isset($first_file)) { $first_file = $file; } $last_file = $file; $frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file); } $previous_file = $file; } } elseif (strpos($file, 'ms_') !== false && strpos($file, '.hist') === false) { $parts = explode('_', $file); if (count($parts) >= 2) { $time = intval($parts[1]) - $startOffset; if ($time >= 0 && (!isset($end) || $time <= $end)) { if (isset($previous_file) && !array_key_exists(0, $frames['frames']) && $time > 0) { $frames['frames'][0] = array('path' => "{$base_path}/{$previous_file}", 'file' => $previous_file); $first_file = $previous_file; } elseif (!isset($first_file)) { $first_file = $file; } $last_file = $file; $frames['frames'][$time] = array('path' => "{$base_path}/{$file}", 'file' => $file); } $previous_file = $file; } } } if (count($frames['frames']) == 1) { foreach ($frames['frames'] as $time => &$frame) { $frame['progress'] = 100; $frames['complete'] = $time; } } elseif (isset($first_file) && strlen($first_file) && isset($last_file) && strlen($last_file) && count($frames['frames'])) { $histograms = null; if (gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json")) { $histograms = json_decode(gz_file_get_contents("{$testPath}/{$run}.{$cached}.histograms.json"), true); } $start_histogram = GetImageHistogram("{$video_directory}/{$first_file}", $options, $histograms); $final_histogram = GetImageHistogram("{$video_directory}/{$last_file}", $options, $histograms); foreach ($frames['frames'] as $time => &$frame) { $histogram = GetImageHistogram("{$video_directory}/{$frame['file']}", $options, $histograms); $frame['progress'] = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5); if ($frame['progress'] == 100 && !array_key_exists('complete', $frames)) { $frames['complete'] = $time; } } } } elseif (gz_is_file("{$testPath}/{$run}.{$cached}.histograms.json")) { $raw = json_decode(gz_file_get_contents("{$testPath}/{$run}.{$cached}.histograms.json"), true); $histograms = array(); foreach ($raw as $h) { if (isset($h['time']) && isset($h['histogram'])) { $histograms[$h['time']] = $h['histogram']; } } ksort($histograms, SORT_NUMERIC); $final_histogram = end($histograms); $start_histogram = reset($histograms); foreach ($histograms as $time => $histogram) { $frames['frames'][$time] = array(); $progress = CalculateFrameProgress($histogram, $start_histogram, $final_histogram, 5); $frames['frames'][$time]['progress'] = $progress; if ($progress == 100 && !isset($frames['complete'])) { $frames['complete'] = $time; } } } } if (isset($frames) && !array_key_exists('SpeedIndex', $frames)) { $dirty = true; $frames['SpeedIndex'] = CalculateSpeedIndex($frames); } if (isset($frames)) { $frames['visualComplete'] = 0; foreach ($frames['frames'] as $time => &$frame) { if ($frame['progress'] > 0 && !array_key_exists('startRender', $frames)) { $frames['startRender'] = $time; } if (!$frames['visualComplete'] && $frame['progress'] == 100) { $frames['visualComplete'] = $time; } // fix up the frame paths in case we have a cached version referencing a relay path if (isset($frame['path'])) { $frame['path'] = $base_path . '/' . basename($frame['path']); } } } if ($completed && !isset($end) && !isset($options) && $dirty && isset($frames) && count($frames)) { gz_file_put_contents($cache_file, json_encode($frames)); } return $frames; }
} $start = microtime(true); // if FreeType isn't supported we can't draw text $gdinfo = gd_info(); if (!isset($gdinfo['FreeType Support']) || !$gdinfo['FreeType Support']) { $labelHeight = 0; $timeHeight = 0; } // Load the information about the video that needs rendering if (isset($_REQUEST['id'])) { $videoId = trim($_REQUEST['id']); $videoPath = './' . GetVideoPath($_REQUEST['id']); if (!is_file("{$videoPath}/video.ini")) { $optionsFile = "{$videoPath}/testinfo.json"; if (gz_is_file($optionsFile)) { $tests = json_decode(gz_file_get_contents($optionsFile), true); if (isset($tests) && !is_array($tests)) { unset($tests); } } } } // Render the video if (isset($tests) && count($tests)) { $lock = Lock("video-{$videoId}", false, 600); if ($lock) { RenderVideo($tests); if (is_file("{$videoPath}/render.mp4")) { rename("{$videoPath}/render.mp4", "{$videoPath}/video.mp4"); } $ini = 'completed=' . gmdate('c') . "\r\n";
/** * Get the relative offset for the video capture (in milliseconds). * This is the time between the first non-timeline event and the * last paint or rasterize event prior to it. * * @param mixed $testPath * @param mixed $run * @param mixed $cached */ function DevToolsGetVideoOffset($testPath, $run, $cached, &$endTime) { $offset = 0; $endTime = 0; $lastEvent = 0; $cachedText = ''; if ($cached) { $cachedText = '_Cached'; } $devToolsFile = "{$testPath}/{$run}{$cachedText}_devtools.json"; if (gz_is_file($devToolsFile)) { $events = json_decode(gz_file_get_contents($devToolsFile), true); if (is_array($events)) { $lastPaint = 0; $startTime = 0; foreach ($events as &$event) { if (is_array($event) && array_key_exists('method', $event)) { $method_class = substr($event['method'], 0, strpos($event['method'], '.')); // calculate the start time stuff if ($method_class === 'Timeline') { $encoded = json_encode($event); $eventTime = DevToolsEventEndTime($event); if ($eventTime && (!$startTime || $eventTime <= $startTime) && (!$lastPaint || $eventTime > $lastPaint)) { if (strpos($encoded, '"type":"ResourceSendRequest"') !== false) { $startTime = DevToolsEventTime($event); } if (strpos($encoded, '"type":"Rasterize"') !== false || strpos($encoded, '"type":"CompositeLayers"') !== false || strpos($encoded, '"type":"Paint"') !== false) { $lastPaint = $eventTime; } } if ($eventTime > $lastEvent && strpos($encoded, '"type":"Resource') !== false) { $lastEvent = $eventTime; } } } } } } if ($startTime && $lastPaint && $lastPaint < $startTime) { $offset = round($startTime - $lastPaint); } if ($startTime && $lastEvent && $lastEvent > $startTime) { $endTime = ceil($lastEvent - $startTime); } return $offset; }