/**
* Gather all of the data that we collect for a single run
*
* @param mixed $id
* @param mixed $testPath
* @param mixed $run
* @param mixed $cached
*/
function GetSingleRunData($id, $testPath, $run, $cached, &$pageData, $testInfo)
{
$ret = null;
if (array_key_exists($run, $pageData) && is_array($pageData[$run]) && array_key_exists($cached, $pageData[$run]) && is_array($pageData[$run][$cached])) {
$protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on' || isset($_SERVER['HTTP_SSL']) && $_SERVER['HTTP_SSL'] == 'On' ? 'https' : 'http';
$host = $_SERVER['HTTP_HOST'];
$uri = rtrim(dirname($_SERVER['PHP_SELF']), '/\\');
$path = substr($testPath, 1);
$ret = $pageData[$run][$cached];
$ret['run'] = $run;
$cachedText = '';
if ($cached) {
$cachedText = '_Cached';
}
if (isset($testInfo)) {
if (array_key_exists('tester', $testInfo)) {
$ret['tester'] = $testInfo['tester'];
}
if (array_key_exists('test_runs', $testInfo) && array_key_exists($run, $testInfo['test_runs']) && array_key_exists('tester', $testInfo['test_runs'][$run])) {
$ret['tester'] = $testInfo['test_runs'][$run]['tester'];
}
}
$basic_results = false;
if (array_key_exists('basic', $_REQUEST) && $_REQUEST['basic']) {
$basic_results = true;
}
if (!$basic_results && gz_is_file("{$testPath}/{$run}{$cachedText}_pagespeed.txt")) {
$ret['PageSpeedScore'] = GetPageSpeedScore("{$testPath}/{$run}{$cachedText}_pagespeed.txt");
$ret['PageSpeedData'] = "{$protocol}://{$host}{$uri}//getgzip.php?test={$id}&file={$run}{$cachedText}_pagespeed.txt";
}
$ret['pages'] = array();
$ret['pages']['details'] = "{$protocol}://{$host}{$uri}/details.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['checklist'] = "{$protocol}://{$host}{$uri}/performance_optimization.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['breakdown'] = "{$protocol}://{$host}{$uri}/breakdown.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['domains'] = "{$protocol}://{$host}{$uri}/domains.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['screenShot'] = "{$protocol}://{$host}{$uri}/screen_shot.php?test={$id}&run={$run}&cached={$cached}";
$ret['thumbnails'] = array();
$ret['thumbnails']['waterfall'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_waterfall_thumb.png";
$ret['thumbnails']['checklist'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_optimization_thumb.png";
$ret['thumbnails']['screenShot'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_screen_thumb.png";
$ret['images'] = array();
$ret['images']['waterfall'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_waterfall.png";
$ret['images']['connectionView'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_connection.png";
$ret['images']['checklist'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_optimization.png";
$ret['images']['screenShot'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.jpg";
if (is_file("{$testPath}/{$run}{$cachedText}_screen.png")) {
$ret['images']['screenShotPng'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.png";
}
$ret['rawData'] = array();
$ret['rawData']['headers'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_report.txt";
$ret['rawData']['pageData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWPG.txt";
$ret['rawData']['requestsData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWTR.txt";
$ret['rawData']['utilization'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_progress.csv";
if (is_file("{$testPath}/{$run}{$cachedText}_bodies.zip")) {
$ret['rawData']['bodies'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_bodies.zip";
}
if (!$basic_results) {
$startOffset = array_key_exists('testStartOffset', $ret) ? intval(round($ret['testStartOffset'])) : 0;
$progress = GetVisualProgress($testPath, $run, $cached, null, null, $startOffset);
if (array_key_exists('frames', $progress) && is_array($progress['frames']) && count($progress['frames'])) {
$cachedTextLower = strtolower($cachedText);
$ret['videoFrames'] = array();
foreach ($progress['frames'] as $ms => $frame) {
$videoFrame = array('time' => $ms);
$videoFrame['image'] = "http://{$host}{$uri}{$path}/video_{$run}{$cachedTextLower}/{$frame['file']}";
$videoFrame['VisuallyComplete'] = $frame['progress'];
$ret['videoFrames'][] = $videoFrame;
}
}
$requests = getRequests($id, $testPath, $run, $cached, $secure, $haveLocations, false, true);
$ret['domains'] = getDomainBreakdown($id, $testPath, $run, $cached, $requests);
$ret['breakdown'] = getBreakdown($id, $testPath, $run, $cached, $requests);
// check if removing requests
$addRequests = 1;
if (isset($_GET['requests'])) {
if ($_GET['requests'] == 0) {
$addRequests = 0;
}
}
// add requests
if ($addRequests == 1) {
$ret['requests'] = $requests;
}
$console_log = DevToolsGetConsoleLog($testPath, $run, $cached);
if (isset($console_log)) {
$ret['consoleLog'] = $console_log;
}
if (gz_is_file("{$testPath}/{$run}{$cachedText}_status.txt")) {
$ret['status'] = array();
$lines = gz_file("{$testPath}/{$run}{$cachedText}_status.txt");
foreach ($lines as $line) {
$line = trim($line);
if (strlen($line)) {
list($time, $message) = explode("\t", $line);
if (strlen($time) && strlen($message)) {
$ret['status'][] = array('time' => $time, 'message' => $message);
}
}
}
}
}
}
return $ret;
}
/**
* Do any aggregation once all of the tests have finished
*
* @param mixed $state
*/
function CollectResults(&$test, &$data)
{
global $logFile;
$testPath = './' . GetTestPath($test['id']);
logMsg("Loading page data from {$testPath}", "./log/{$logFile}", true);
$page_data = loadAllPageData($testPath);
if (count($page_data)) {
foreach ($page_data as $run => &$page_run) {
foreach ($page_run as $cached => &$test_data) {
$data_row = $test_data;
unset($data_row['URL']);
// figure out the per-type request info (todo: measure how expensive this is and see if we have a better way)
$breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$data_row["{$mime}_requests"] = $values['requests'];
$data_row["{$mime}_bytes"] = $values['bytes'];
}
// capture the page speed score
if ($cached) {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt");
} else {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt");
}
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['cached'] = $cached;
$data_row['run'] = $run;
$data_row['id'] = $test['id'];
$data[] = $data_row;
$test['has_data'] = 1;
}
}
} else {
$data_row = array();
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['id'] = $test['id'];
$data[] = $data_row;
}
}
$tester = $test['testinfo']['tester'];
}
if (array_key_exists('test_runs', $test['testinfo']) && array_key_exists($i, $test['testinfo']['test_runs']) && array_key_exists('tester', $test['testinfo']['test_runs'][$i])) {
$tester = $test['testinfo']['test_runs'][$i]['tester'] . '<br>';
}
if (isset($tester)) {
echo "<tester>" . xml_entities($tester) . "</tester>\n";
}
}
echo "<results>\n";
foreach ($pageData[$i][1] as $key => $val) {
$key = preg_replace('/[^a-zA-Z0-9\\.\\-_]/', '_', $key);
echo "<{$key}>" . xml_entities($val) . "</{$key}>\n";
}
if ($pagespeed) {
$score = GetPageSpeedScore("{$testPath}/{$i}_Cached_pagespeed.txt");
if (strlen($score)) {
echo "<PageSpeedScore>{$score}</PageSpeedScore>\n";
}
}
echo "</results>\n";
// links to the relevant pages
echo "<pages>\n";
echo "<details>http://{$host}{$uri}/result/{$id}/{$i}/details/cached/</details>\n";
echo "<checklist>http://{$host}{$uri}/result/{$id}/{$i}/performance_optimization/cached/</checklist>\n";
echo "<report>http://{$host}{$uri}/result/{$id}/{$i}/optimization_report/cached/</report>\n";
echo "<breakdown>http://{$host}{$uri}/result/{$id}/{$i}/breakdown/</breakdown>\n";
echo "<domains>http://{$host}{$uri}/result/{$id}/{$i}/domains/</domains>\n";
echo "<screenShot>http://{$host}{$uri}/result/{$id}/{$i}/screen_shot/cached/</screenShot>\n";
echo "</pages>\n";
// urls for the relevant images
function CollectTestResult($test, &$data)
{
global $benchmark;
$id = $test['id'];
$count = 0;
echo "Reprocessing Test {$id}...";
logMsg("Reprocessing Test {$id}", "./log/reprocess-{$benchmark}.log", true);
RestoreTest($id);
ReprocessVideo($id);
$testPath = './' . GetTestPath($id);
$page_data = loadAllPageData($testPath);
if (count($page_data)) {
foreach ($page_data as $run => &$page_run) {
foreach ($page_run as $cached => &$test_data) {
$data_row = $test_data;
unset($data_row['URL']);
// figure out the per-type request info (todo: measure how expensive this is and see if we have a better way)
$breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$data_row["{$mime}_requests"] = $values['requests'];
$data_row["{$mime}_bytes"] = $values['bytes'];
}
// capture the page speed score
if ($cached) {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt");
} else {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt");
}
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['cached'] = $cached;
$data_row['run'] = $run;
$data_row['id'] = $test['id'];
$data[] = $data_row;
$count++;
}
}
} else {
$data_row = array();
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['id'] = $test['id'];
$data[] = $data_row;
}
// If the test was already archived, re-archive it.
$testInfo = GetTestInfo($id);
if (array_key_exists('archived', $testInfo) && $testInfo['archived']) {
$lock = LockTest($id);
if (isset($lock)) {
$testInfo = GetTestInfo($id);
$testInfo['archived'] = false;
SaveTestInfo($id, $testInfo);
UnlockTest($lock);
}
ArchiveTest($id);
}
echo "{$count} results\n";
}
/**
* Build the data set
*
* @param mixed $pageData
*/
function BuildHAR(&$pageData, $id, $testPath, $options)
{
$result = array();
$entries = array();
$result['log'] = array();
$result['log']['version'] = '1.1';
$result['log']['creator'] = array('name' => 'WebPagetest', 'version' => VER_WEBPAGETEST);
$result['log']['pages'] = array();
foreach ($pageData as $run => $pageRun) {
foreach ($pageRun as $cached => $data) {
$cached_text = '';
if ($cached) {
$cached_text = '_Cached';
}
if (!array_key_exists('browser', $result['log'])) {
$result['log']['browser'] = array('name' => $data['browser_name'], 'version' => $data['browser_version']);
}
$pd = array();
$pd['startedDateTime'] = msdate($data['date']);
$pd['title'] = "Run {$run}, ";
if ($cached) {
$pd['title'] .= "Repeat View";
} else {
$pd['title'] .= "First View";
}
$pd['title'] .= " for " . $data['URL'];
$pd['id'] = "page_{$run}_{$cached}";
$pd['pageTimings'] = array('onLoad' => $data['docTime'], 'onContentLoad' => -1, '_startRender' => $data['render']);
// add the pagespeed score
if (gz_is_file("{$testPath}/{$run}{$cached_text}_pagespeed.txt")) {
$pagespeed_data = LoadPageSpeedData("{$testPath}/{$run}{$cached_text}_pagespeed.txt");
if ($pagespeed_data) {
$score = GetPageSpeedScore(null, $pagespeed_data);
if (strlen($score)) {
$pd['_pageSpeed'] = array('score' => $score, 'result' => $pagespeed_data);
}
}
}
// dump all of our metrics into the har data as custom fields
foreach ($data as $name => $value) {
if (!is_array($value)) {
$pd["_{$name}"] = $value;
}
}
// add the page-level ldata to the result
$result['log']['pages'][] = $pd;
// now add the object-level data to the result
$secure = false;
$haveLocations = false;
$requests = getRequests($id, $testPath, $run, $cached, $secure, $haveLocations, false, true);
foreach ($requests as &$r) {
$entry = array();
$entry['pageref'] = $pd['id'];
$entry['startedDateTime'] = msdate((double) $data['date'] + $r['load_start'] / 1000.0);
$entry['time'] = $r['all_ms'];
$request = array();
$request['method'] = $r['method'];
$protocol = $r['is_secure'] ? 'https://' : 'http://';
$request['url'] = $protocol . $r['host'] . $r['url'];
$request['headersSize'] = -1;
$request['bodySize'] = -1;
$request['cookies'] = array();
$request['headers'] = array();
$ver = '';
$headersSize = 0;
if (isset($r['headers']) && isset($r['headers']['request'])) {
foreach ($r['headers']['request'] as &$header) {
$headersSize += strlen($header) + 2;
// add 2 for the \r\n that is on the raw headers
$pos = strpos($header, ':');
if ($pos > 0) {
$name = trim(substr($header, 0, $pos));
$val = trim(substr($header, $pos + 1));
if (strlen($name)) {
$request['headers'][] = array('name' => $name, 'value' => $val);
}
// parse out any cookies
if (!strcasecmp($name, 'cookie')) {
$cookies = explode(';', $val);
foreach ($cookies as &$cookie) {
$pos = strpos($cookie, '=');
if ($pos > 0) {
$name = (string) trim(substr($cookie, 0, $pos));
$val = (string) trim(substr($cookie, $pos + 1));
if (strlen($name)) {
$request['cookies'][] = array('name' => $name, 'value' => $val);
}
}
}
}
} else {
$pos = strpos($header, 'HTTP/');
if ($pos >= 0) {
$ver = (string) trim(substr($header, $pos + 5, 3));
}
}
}
}
if ($headersSize) {
$request['headersSize'] = $headersSize;
}
$request['httpVersion'] = $ver;
$request['queryString'] = array();
$parts = parse_url($request['url']);
if (isset($parts['query'])) {
$qs = array();
parse_str($parts['query'], $qs);
foreach ($qs as $name => $val) {
$request['queryString'][] = array('name' => (string) $name, 'value' => (string) $val);
}
}
if (!strcasecmp(trim($request['method']), 'post')) {
$request['postData'] = array();
$request['postData']['mimeType'] = '';
$request['postData']['text'] = '';
}
$entry['request'] = $request;
$response = array();
$response['status'] = (int) $r['responseCode'];
$response['statusText'] = '';
$response['headersSize'] = -1;
$response['bodySize'] = (int) $r['objectSize'];
$response['headers'] = array();
$ver = '';
$loc = '';
$headersSize = 0;
if (isset($r['headers']) && isset($r['headers']['response'])) {
foreach ($r['headers']['response'] as &$header) {
$headersSize += strlen($header) + 2;
// add 2 for the \r\n that is on the raw headers
$pos = strpos($header, ':');
if ($pos > 0) {
$name = (string) trim(substr($header, 0, $pos));
$val = (string) trim(substr($header, $pos + 1));
if (strlen($name)) {
$response['headers'][] = array('name' => $name, 'value' => $val);
}
if (!strcasecmp($name, 'location')) {
$loc = (string) $val;
}
} else {
$pos = strpos($header, 'HTTP/');
if ($pos >= 0) {
$ver = (string) trim(substr($header, $pos + 5, 3));
}
}
}
}
if ($headersSize) {
$response['headersSize'] = $headersSize;
}
$response['httpVersion'] = $ver;
$response['redirectURL'] = $loc;
$response['content'] = array();
$response['content']['size'] = (int) $r['objectSize'];
if (isset($r['contentType']) && strlen($r['contentType'])) {
$response['content']['mimeType'] = (string) $r['contentType'];
} else {
$response['content']['mimeType'] = '';
}
// unsupported fields that are required
$response['cookies'] = array();
$entry['response'] = $response;
$entry['cache'] = (object) array();
$timings = array();
$timings['blocked'] = -1;
$timings['dns'] = (int) $r['dns_ms'];
if (!$timings['dns']) {
$timings['dns'] = -1;
}
// HAR did not have an ssl time until version 1.2 . For
// backward compatibility, "connect" includes "ssl" time.
// WepbageTest's internal representation does not assume any
// overlap, so we must add our connect and ssl time to get the
// connect time expected by HAR.
$timings['connect'] = durationOfInterval($r['connect_ms']) + durationOfInterval($r['ssl_ms']);
if (!$timings['connect']) {
$timings['connect'] = -1;
}
$timings['ssl'] = (int) $r['ssl_ms'];
if (!$timings['ssl']) {
$timings['ssl'] = -1;
}
// TODO(skerner): WebpageTest's data model has no way to
// represent the difference between the states HAR calls
// send (time required to send HTTP request to the server)
// and wait (time spent waiting for a response from the server).
// We lump both into "wait". Issue 24* tracks this work. When
// it is resolved, read the real values for send and wait
// instead of using the request's TTFB.
// *: http://code.google.com/p/webpagetest/issues/detail?id=24
$timings['send'] = 0;
$timings['wait'] = (int) $r['ttfb_ms'];
$timings['receive'] = (int) $r['download_ms'];
$entry['timings'] = $timings;
// The HAR spec defines time as the sum of the times in the
// timings object, excluding any unknown (-1) values and ssl
// time (which is included in "connect", for backward
// compatibility with tools written before "ssl" was defined
// in HAR version 1.2).
$entry['time'] = 0;
foreach ($timings as $timingKey => $duration) {
if ($timingKey != 'ssl' && $duration != UNKNOWN_TIME) {
$entry['time'] += $duration;
}
}
if (array_key_exists('custom_rules', $r)) {
$entry['_custom_rules'] = $r['custom_rules'];
}
// dump all of our metrics into the har data as custom fields
foreach ($r as $name => $value) {
if (!is_array($value)) {
$entry["_{$name}"] = $value;
}
}
// add it to the list of entries
$entries[] = $entry;
}
// add the bodies to the requests
if (isset($options['bodies']) && $options['bodies']) {
$bodies_file = $testPath . '/' . $run . $cached_text . '_bodies.zip';
if (is_file($bodies_file)) {
$zip = new ZipArchive();
if ($zip->open($bodies_file) === TRUE) {
for ($i = 0; $i < $zip->numFiles; $i++) {
$index = intval($zip->getNameIndex($i), 10) - 1;
if (array_key_exists($index, $entries)) {
$entries[$index]['response']['content']['text'] = utf8_encode($zip->getFromIndex($i));
}
}
}
}
}
}
}
$result['log']['entries'] = $entries;
return $result;
}
/**
* @return string The score
*/
public function getPageSpeedScore()
{
// TODO: move implementation to this method
if ($this->fileHandler->gzFileExists($this->localPaths->pageSpeedFile())) {
return GetPageSpeedScore($this->localPaths->pageSpeedFile());
}
return null;
}
