/**
* Gather all of the data that we collect for a single run
*
* @param mixed $id
* @param mixed $testPath
* @param mixed $run
* @param mixed $cached
*/
function GetSingleRunData($id, $testPath, $run, $cached, &$pageData, $testInfo)
{
$ret = null;
if (array_key_exists($run, $pageData) && is_array($pageData[$run]) && array_key_exists($cached, $pageData[$run]) && is_array($pageData[$run][$cached])) {
$protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on' || isset($_SERVER['HTTP_SSL']) && $_SERVER['HTTP_SSL'] == 'On' ? 'https' : 'http';
$host = $_SERVER['HTTP_HOST'];
$uri = rtrim(dirname($_SERVER['PHP_SELF']), '/\\');
$path = substr($testPath, 1);
$ret = $pageData[$run][$cached];
$ret['run'] = $run;
$cachedText = '';
if ($cached) {
$cachedText = '_Cached';
}
if (isset($testInfo)) {
if (array_key_exists('tester', $testInfo)) {
$ret['tester'] = $testInfo['tester'];
}
if (array_key_exists('test_runs', $testInfo) && array_key_exists($run, $testInfo['test_runs']) && array_key_exists('tester', $testInfo['test_runs'][$run])) {
$ret['tester'] = $testInfo['test_runs'][$run]['tester'];
}
}
$basic_results = false;
if (array_key_exists('basic', $_REQUEST) && $_REQUEST['basic']) {
$basic_results = true;
}
if (!$basic_results && gz_is_file("{$testPath}/{$run}{$cachedText}_pagespeed.txt")) {
$ret['PageSpeedScore'] = GetPageSpeedScore("{$testPath}/{$run}{$cachedText}_pagespeed.txt");
$ret['PageSpeedData'] = "{$protocol}://{$host}{$uri}//getgzip.php?test={$id}&file={$run}{$cachedText}_pagespeed.txt";
}
$ret['pages'] = array();
$ret['pages']['details'] = "{$protocol}://{$host}{$uri}/details.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['checklist'] = "{$protocol}://{$host}{$uri}/performance_optimization.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['breakdown'] = "{$protocol}://{$host}{$uri}/breakdown.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['domains'] = "{$protocol}://{$host}{$uri}/domains.php?test={$id}&run={$run}&cached={$cached}";
$ret['pages']['screenShot'] = "{$protocol}://{$host}{$uri}/screen_shot.php?test={$id}&run={$run}&cached={$cached}";
$ret['thumbnails'] = array();
$ret['thumbnails']['waterfall'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_waterfall_thumb.png";
$ret['thumbnails']['checklist'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_optimization_thumb.png";
$ret['thumbnails']['screenShot'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_screen_thumb.png";
$ret['images'] = array();
$ret['images']['waterfall'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_waterfall.png";
$ret['images']['connectionView'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_connection.png";
$ret['images']['checklist'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_optimization.png";
$ret['images']['screenShot'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.jpg";
if (is_file("{$testPath}/{$run}{$cachedText}_screen.png")) {
$ret['images']['screenShotPng'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.png";
}
$ret['rawData'] = array();
$ret['rawData']['headers'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_report.txt";
$ret['rawData']['pageData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWPG.txt";
$ret['rawData']['requestsData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWTR.txt";
$ret['rawData']['utilization'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_progress.csv";
if (is_file("{$testPath}/{$run}{$cachedText}_bodies.zip")) {
$ret['rawData']['bodies'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_bodies.zip";
}
if (!$basic_results) {
$startOffset = array_key_exists('testStartOffset', $ret) ? intval(round($ret['testStartOffset'])) : 0;
$progress = GetVisualProgress($testPath, $run, $cached, null, null, $startOffset);
if (array_key_exists('frames', $progress) && is_array($progress['frames']) && count($progress['frames'])) {
$cachedTextLower = strtolower($cachedText);
$ret['videoFrames'] = array();
foreach ($progress['frames'] as $ms => $frame) {
$videoFrame = array('time' => $ms);
$videoFrame['image'] = "http://{$host}{$uri}{$path}/video_{$run}{$cachedTextLower}/{$frame['file']}";
$videoFrame['VisuallyComplete'] = $frame['progress'];
$ret['videoFrames'][] = $videoFrame;
}
}
$requests = getRequests($id, $testPath, $run, $cached, $secure, $haveLocations, false, true);
$ret['domains'] = getDomainBreakdown($id, $testPath, $run, $cached, $requests);
$ret['breakdown'] = getBreakdown($id, $testPath, $run, $cached, $requests);
// check if removing requests
$addRequests = 1;
if (isset($_GET['requests'])) {
if ($_GET['requests'] == 0) {
$addRequests = 0;
}
}
// add requests
if ($addRequests == 1) {
$ret['requests'] = $requests;
}
$console_log = DevToolsGetConsoleLog($testPath, $run, $cached);
if (isset($console_log)) {
$ret['consoleLog'] = $console_log;
}
if (gz_is_file("{$testPath}/{$run}{$cachedText}_status.txt")) {
$ret['status'] = array();
$lines = gz_file("{$testPath}/{$run}{$cachedText}_status.txt");
foreach ($lines as $line) {
$line = trim($line);
if (strlen($line)) {
list($time, $message) = explode("\t", $line);
if (strlen($time) && strlen($message)) {
$ret['status'][] = array('time' => $time, 'message' => $message);
}
}
}
}
}
}
return $ret;
}
/**
* Do any aggregation once all of the tests have finished
*
* @param mixed $state
*/
function CollectResults(&$test, &$data)
{
global $logFile;
$testPath = './' . GetTestPath($test['id']);
logMsg("Loading page data from {$testPath}", "./log/{$logFile}", true);
$page_data = loadAllPageData($testPath);
if (count($page_data)) {
foreach ($page_data as $run => &$page_run) {
foreach ($page_run as $cached => &$test_data) {
$data_row = $test_data;
unset($data_row['URL']);
// figure out the per-type request info (todo: measure how expensive this is and see if we have a better way)
$breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$data_row["{$mime}_requests"] = $values['requests'];
$data_row["{$mime}_bytes"] = $values['bytes'];
}
// capture the page speed score
if ($cached) {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt");
} else {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt");
}
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['cached'] = $cached;
$data_row['run'] = $run;
$data_row['id'] = $test['id'];
$data[] = $data_row;
$test['has_data'] = 1;
}
}
} else {
$data_row = array();
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['id'] = $test['id'];
$data[] = $data_row;
}
}
if (!array_key_exists('errors', $testInfo)) {
$testInfo['errors'] = array();
}
if (!array_key_exists($runNumber, $testInfo['errors'])) {
$testInfo['errors'][$runNumber] = array();
}
$testInfo['errors'][$runNumber][$cacheWarmed] = $_REQUEST['error'];
$testInfo_dirty = true;
}
// Do any post-processing on this individual run that doesn't requre the test to be locked
if (isset($runNumber) && isset($cacheWarmed)) {
$secure = false;
$haveLocations = false;
$requests = getRequests($id, $testPath, $runNumber, $cacheWarmed, $secure, $haveLocations, false);
if (isset($requests) && is_array($requests) && count($requests)) {
getBreakdown($id, $testPath, $runNumber, $cacheWarmed, $requests);
} else {
$testerError = 'Missing Results';
}
if (is_dir('./google') && is_file('./google/google_lib.inc')) {
require_once 'google/google_lib.inc';
ParseCsiInfo($id, $testPath, $runNumber, $cacheWarmed, true);
}
GetDevToolsCPUTime($testPath, $runNumber, $cacheWarmed);
}
// mark this run as complete
if (isset($runNumber) && isset($cacheWarmed)) {
if ($testInfo['fvonly'] || $cacheWarmed) {
if (!array_key_exists('test_runs', $testInfo)) {
$testInfo['test_runs'] = array();
}
/**
* Display the comparison graph with the various time metrics
*
*/
function DisplayGraphs()
{
global $tests;
global $filmstrip_end_frame;
require_once 'breakdown.inc';
$mimeTypes = array('html', 'js', 'css', 'image', 'flash', 'font', 'other');
$timeMetrics = array('visualComplete' => 'Visually Complete', 'lastVisualChange' => 'Last Visual Change', 'docTime' => 'Load Time (onload)', 'fullyLoaded' => 'Load Time (Fully Loaded)', 'domContentLoadedEventStart' => 'DOM Content Loaded', 'SpeedIndex' => 'Speed Index', 'TTFB' => 'Time to First Byte', 'titleTime' => 'Time to Title', 'render' => 'Time to Start Render', 'fullyLoadedCPUms' => 'CPU Busy Time');
$progress_end = 0;
$testCount = count($tests);
foreach ($tests as &$test) {
$requests;
$test['breakdown'] = getBreakdown($test['id'], $test['path'], $test['run'], $test['cached'], $requests);
if (array_key_exists('progress', $test['video']) && array_key_exists('frames', $test['video']['progress'])) {
foreach ($test['video']['progress']['frames'] as $ms => &$data) {
if ($ms > $progress_end && array_key_exists('progress', $data)) {
$progress_end = $ms;
}
}
}
}
if ($progress_end) {
if ($progress_end % 100) {
$progress_end = intval((intval($progress_end / 100) + 1) * 100);
}
echo '<div id="compare_visual_progress" class="compare-graph"></div>';
}
if (count($tests) <= 4) {
echo '<div id="compare_times" class="compare-graph"></div>';
echo '<div id="compare_requests" class="compare-graph"></div>';
echo '<div id="compare_bytes" class="compare-graph"></div>';
} else {
foreach ($timeMetrics as $metric => $label) {
echo "<div id=\"compare_times_{$metric}\" class=\"compare-graph\"></div>";
}
foreach ($mimeTypes as $type) {
echo "<div id=\"compare_requests_{$type}\" class=\"compare-graph\"></div>";
echo "<div id=\"compare_bytes_{$type}\" class=\"compare-graph\"></div>";
}
}
?>
<script type="text/javascript" src="<?php
echo $GLOBALS['ptotocol'];
?>
://www.google.com/jsapi"></script>
<script type="text/javascript">
google.load('visualization', '1', {'packages':['table', 'corechart']});
google.setOnLoadCallback(drawCharts);
function drawCharts() {
var dataTimes = new google.visualization.DataTable();
var dataRequests = new google.visualization.DataTable();
var dataBytes = new google.visualization.DataTable();
dataTimes.addColumn('string', 'Time (ms)');
dataRequests.addColumn('string', 'MIME Type');
dataBytes.addColumn('string', 'MIME Type');
<?php
foreach ($tests as &$test) {
$name = htmlspecialchars($test['name']);
echo "dataTimes.addColumn('number', '{$name}');\n";
echo "dataRequests.addColumn('number', '{$name}');\n";
echo "dataBytes.addColumn('number', '{$name}');\n";
}
echo 'dataTimes.addRows(' . count($timeMetrics) . ");\n";
echo 'dataRequests.addRows(' . strval(count($mimeTypes) + 1) . ");\n";
echo 'dataBytes.addRows(' . strval(count($mimeTypes) + 1) . ");\n";
if ($progress_end) {
echo "var dataProgress = google.visualization.arrayToDataTable([\n";
echo " ['Time (ms)'";
foreach ($tests as &$test) {
echo ", '" . htmlspecialchars($test['name']) . "'";
}
echo " ]";
for ($ms = 0; $ms <= $progress_end; $ms += 100) {
echo ",\n ['" . number_format($ms / 1000, 1) . "'";
foreach ($tests as &$test) {
$progress = 0;
if (array_key_exists('last_progress', $test)) {
$progress = $test['last_progress'];
}
if (array_key_exists('progress', $test['video']) && array_key_exists('frames', $test['video']['progress']) && array_key_exists($ms, $test['video']['progress']['frames'])) {
$progress = $test['video']['progress']['frames'][$ms]['progress'];
}
$test['last_progress'] = $progress;
if (array_key_exists('video', $test) && array_key_exists('progress', $test['video']) && array_key_exists('frames', $test['video']['progress'])) {
foreach ($test['video']['progress']['frames'] as $time => $frameInfo) {
if ($time <= $ms) {
$progress = floatval($frameInfo['progress']);
}
}
}
echo ", {$progress}";
}
echo "]";
}
echo "]);\n";
}
$row = 0;
foreach ($timeMetrics as $metric => $label) {
echo "dataTimes.setValue({$row}, 0, '{$label}');\n";
$column = 1;
foreach ($tests as &$test) {
if (array_key_exists('pageData', $test) && array_key_exists('run', $test) && array_key_exists($test['run'], $test['pageData']) && array_key_exists('cached', $test) && array_key_exists($test['cached'], $test['pageData'][$test['run']]) && array_key_exists($metric, $test['pageData'][$test['run']][$test['cached']]) && strlen($test['pageData'][$test['run']][$test['cached']][$metric])) {
echo "dataTimes.setValue({$row}, {$column}, {$test['pageData'][$test['run']][$test['cached']][$metric]});\n";
}
$column++;
}
$row++;
}
$row = 0;
foreach ($timeMetrics as $metric => $label) {
echo "var dataTimes{$metric} = new google.visualization.DataView(dataTimes);\n";
echo "dataTimes{$metric}.setRows({$row}, {$row});\n";
$row++;
}
echo "dataRequests.setValue(0, 0, 'Total');\n";
echo "dataBytes.setValue(0, 0, 'Total');\n";
$column = 1;
foreach ($tests as &$test) {
if (array_key_exists('pageData', $test) && array_key_exists('run', $test) && array_key_exists($test['run'], $test['pageData']) && array_key_exists('cached', $test) && array_key_exists($test['cached'], $test['pageData'][$test['run']])) {
if (array_key_exists('requests', $test['pageData'][$test['run']][$test['cached']]) && strlen($test['pageData'][$test['run']][$test['cached']]['requests'])) {
echo "dataRequests.setValue(0, {$column}, {$test['pageData'][$test['run']][$test['cached']]['requests']});\n";
}
if (array_key_exists('bytesIn', $test['pageData'][$test['run']][$test['cached']]) && strlen($test['pageData'][$test['run']][$test['cached']]['bytesIn'])) {
echo "dataBytes.setValue(0, {$column}, {$test['pageData'][$test['run']][$test['cached']]['bytesIn']});\n";
}
}
$column++;
}
$row = 1;
foreach ($mimeTypes as $mimeType) {
echo "dataRequests.setValue({$row}, 0, '{$mimeType}');\n";
echo "dataBytes.setValue({$row}, 0, '{$mimeType}');\n";
$column = 1;
foreach ($tests as &$test) {
echo "dataRequests.setValue({$row}, {$column}, {$test['breakdown'][$mimeType]['requests']});\n";
echo "dataBytes.setValue({$row}, {$column}, {$test['breakdown'][$mimeType]['bytes']});\n";
$column++;
}
$row++;
}
$row = 1;
foreach ($mimeTypes as $mimeType) {
echo "var dataRequests{$mimeType} = new google.visualization.DataView(dataRequests);\n";
echo "dataRequests{$mimeType}.setRows({$row}, {$row});\n";
echo "var dataBytes{$mimeType} = new google.visualization.DataView(dataBytes);\n";
echo "dataBytes{$mimeType}.setRows({$row}, {$row});\n";
$row++;
}
if ($progress_end) {
echo "var progressChart = new google.visualization.LineChart(document.getElementById('compare_visual_progress'));\n";
echo "progressChart.draw(dataProgress, {title: 'Visual Progress (%)', hAxis: {title: 'Time (seconds)'}});\n";
}
if (count($tests) <= 4) {
echo "var timesChart = new google.visualization.ColumnChart(document.getElementById('compare_times'));\n";
echo "timesChart.draw(dataTimes, {title: 'Timings (ms)'});\n";
echo "var requestsChart = new google.visualization.ColumnChart(document.getElementById('compare_requests'));\n";
echo "requestsChart.draw(dataRequests, {title: 'Requests'});\n";
echo "var bytesChart = new google.visualization.ColumnChart(document.getElementById('compare_bytes'));\n";
echo "bytesChart.draw(dataBytes, {title: 'Bytes'});\n";
} else {
foreach ($timeMetrics as $metric => $label) {
echo "var timesChart{$metric} = new google.visualization.ColumnChart(document.getElementById('compare_times_{$metric}'));\n";
echo "timesChart{$metric}.draw(dataTimes{$metric}, {title: '{$label} (ms)'});\n";
}
foreach ($mimeTypes as $type) {
echo "var requestsChart{$type} = new google.visualization.ColumnChart(document.getElementById('compare_requests_{$type}'));\n";
echo "requestsChart{$type}.draw(dataRequests{$type}, {title: '{$type} Requests'});\n";
echo "var bytesChart{$type} = new google.visualization.ColumnChart(document.getElementById('compare_bytes_{$type}'));\n";
echo "bytesChart{$type}.draw(dataBytes{$type}, {title: '{$type} Bytes'});\n";
}
}
?>
}
</script>
<?php
}
/**
* Dump a breakdown of the requests and bytes by mime type
*/
function xmlBreakdown($id, $testPath, $run, $cached)
{
if (array_key_exists('breakdown', $_REQUEST) && $_REQUEST['breakdown']) {
echo "<breakdown>\n";
$requests;
$breakdown = getBreakdown($id, $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$domain = strrev($domain);
echo "<{$mime}>\n";
echo "<requests>{$values['requests']}</requests>\n";
echo "<bytes>{$values['bytes']}</bytes>\n";
echo "</{$mime}>\n";
}
echo "</breakdown>\n";
}
}
require_once 'contentColors.inc';
require_once 'waterfall.inc';
require_once 'page_data.inc';
$page_keywords = array('Content Breakdown', 'MIME Types', 'Webpagetest', 'Website Speed Test', 'Page Speed');
$page_description = "Website content breakdown by mime type{$testLabel}";
$extension = 'php';
if (FRIENDLY_URLS) {
$extension = 'png';
}
// walk through the requests and group them by mime type
$requestsFv;
$breakdownFv = getBreakdown($id, $testPath, $run, 0, $requestsFv);
$breakdownRv = array();
$requestsRv = array();
if ((int) $test[test][fvonly] == 0) {
$breakdownRv = getBreakdown($id, $testPath, $run, 1, $requestsRv);
}
?>
<!DOCTYPE html>
<html>
<head>
<title>WebPagetest Content Breakdown<?php
echo $testLabel;
?>
</title>
<?php
$gaTemplate = 'Content Breakdown';
include 'head.inc';
?>
<style type="text/css">
td {
function CollectTestResult($test, &$data)
{
global $benchmark;
$id = $test['id'];
$count = 0;
echo "Reprocessing Test {$id}...";
logMsg("Reprocessing Test {$id}", "./log/reprocess-{$benchmark}.log", true);
RestoreTest($id);
ReprocessVideo($id);
$testPath = './' . GetTestPath($id);
$page_data = loadAllPageData($testPath);
if (count($page_data)) {
foreach ($page_data as $run => &$page_run) {
foreach ($page_run as $cached => &$test_data) {
$data_row = $test_data;
unset($data_row['URL']);
// figure out the per-type request info (todo: measure how expensive this is and see if we have a better way)
$breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$data_row["{$mime}_requests"] = $values['requests'];
$data_row["{$mime}_bytes"] = $values['bytes'];
}
// capture the page speed score
if ($cached) {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt");
} else {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt");
}
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['cached'] = $cached;
$data_row['run'] = $run;
$data_row['id'] = $test['id'];
$data[] = $data_row;
$count++;
}
}
} else {
$data_row = array();
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['id'] = $test['id'];
$data[] = $data_row;
}
// If the test was already archived, re-archive it.
$testInfo = GetTestInfo($id);
if (array_key_exists('archived', $testInfo) && $testInfo['archived']) {
$lock = LockTest($id);
if (isset($lock)) {
$testInfo = GetTestInfo($id);
$testInfo['archived'] = false;
SaveTestInfo($id, $testInfo);
UnlockTest($lock);
}
ArchiveTest($id);
}
echo "{$count} results\n";
}
JpGraphError::SetErrLocale('prod');
// Get the various settings
$width = htmlspecialchars($_GET["width"]);
if (!$width) {
$width = 500;
}
$height = htmlspecialchars($_GET["height"]);
if (!$height) {
$height = 300;
}
$chartType = htmlspecialchars($_GET["type"]);
if (!strlen($chartType)) {
$chartType = 'Requests';
}
// walk through the requests and group them by mime type
$breakdown = getBreakdown($id, $testPath, $run, $cached);
if (count($breakdown)) {
// build up the data set
$values = array();
$index = 0;
ksort($breakdown);
foreach ($breakdown as $data) {
if (!strcasecmp($chartType, 'Requests')) {
$values[] = $data['requests'];
} else {
if (!strcasecmp($chartType, 'Bytes')) {
$values[] = $data['bytes'];
}
}
$index++;
}
