/** * Gather all of the data that we collect for a single run * * @param mixed $id * @param mixed $testPath * @param mixed $run * @param mixed $cached */ function GetSingleRunData($id, $testPath, $run, $cached, &$pageData, $testInfo) { $ret = null; if (array_key_exists($run, $pageData) && is_array($pageData[$run]) && array_key_exists($cached, $pageData[$run]) && is_array($pageData[$run][$cached])) { $protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on' || isset($_SERVER['HTTP_SSL']) && $_SERVER['HTTP_SSL'] == 'On' ? 'https' : 'http'; $host = $_SERVER['HTTP_HOST']; $uri = rtrim(dirname($_SERVER['PHP_SELF']), '/\\'); $path = substr($testPath, 1); $ret = $pageData[$run][$cached]; $ret['run'] = $run; $cachedText = ''; if ($cached) { $cachedText = '_Cached'; } if (isset($testInfo)) { if (array_key_exists('tester', $testInfo)) { $ret['tester'] = $testInfo['tester']; } if (array_key_exists('test_runs', $testInfo) && array_key_exists($run, $testInfo['test_runs']) && array_key_exists('tester', $testInfo['test_runs'][$run])) { $ret['tester'] = $testInfo['test_runs'][$run]['tester']; } } $basic_results = false; if (array_key_exists('basic', $_REQUEST) && $_REQUEST['basic']) { $basic_results = true; } if (!$basic_results && gz_is_file("{$testPath}/{$run}{$cachedText}_pagespeed.txt")) { $ret['PageSpeedScore'] = GetPageSpeedScore("{$testPath}/{$run}{$cachedText}_pagespeed.txt"); $ret['PageSpeedData'] = "{$protocol}://{$host}{$uri}//getgzip.php?test={$id}&file={$run}{$cachedText}_pagespeed.txt"; } $ret['pages'] = array(); $ret['pages']['details'] = "{$protocol}://{$host}{$uri}/details.php?test={$id}&run={$run}&cached={$cached}"; $ret['pages']['checklist'] = "{$protocol}://{$host}{$uri}/performance_optimization.php?test={$id}&run={$run}&cached={$cached}"; $ret['pages']['breakdown'] = "{$protocol}://{$host}{$uri}/breakdown.php?test={$id}&run={$run}&cached={$cached}"; $ret['pages']['domains'] = "{$protocol}://{$host}{$uri}/domains.php?test={$id}&run={$run}&cached={$cached}"; $ret['pages']['screenShot'] = "{$protocol}://{$host}{$uri}/screen_shot.php?test={$id}&run={$run}&cached={$cached}"; $ret['thumbnails'] = array(); $ret['thumbnails']['waterfall'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_waterfall_thumb.png"; $ret['thumbnails']['checklist'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_optimization_thumb.png"; $ret['thumbnails']['screenShot'] = "{$protocol}://{$host}{$uri}/result/{$id}/{$run}{$cachedText}_screen_thumb.png"; $ret['images'] = array(); $ret['images']['waterfall'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_waterfall.png"; $ret['images']['connectionView'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_connection.png"; $ret['images']['checklist'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_optimization.png"; $ret['images']['screenShot'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.jpg"; if (is_file("{$testPath}/{$run}{$cachedText}_screen.png")) { $ret['images']['screenShotPng'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_screen.png"; } $ret['rawData'] = array(); $ret['rawData']['headers'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_report.txt"; $ret['rawData']['pageData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWPG.txt"; $ret['rawData']['requestsData'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_IEWTR.txt"; $ret['rawData']['utilization'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_progress.csv"; if (is_file("{$testPath}/{$run}{$cachedText}_bodies.zip")) { $ret['rawData']['bodies'] = "{$protocol}://{$host}{$uri}{$path}/{$run}{$cachedText}_bodies.zip"; } if (!$basic_results) { $startOffset = array_key_exists('testStartOffset', $ret) ? intval(round($ret['testStartOffset'])) : 0; $progress = GetVisualProgress($testPath, $run, $cached, null, null, $startOffset); if (array_key_exists('frames', $progress) && is_array($progress['frames']) && count($progress['frames'])) { $cachedTextLower = strtolower($cachedText); $ret['videoFrames'] = array(); foreach ($progress['frames'] as $ms => $frame) { $videoFrame = array('time' => $ms); $videoFrame['image'] = "http://{$host}{$uri}{$path}/video_{$run}{$cachedTextLower}/{$frame['file']}"; $videoFrame['VisuallyComplete'] = $frame['progress']; $ret['videoFrames'][] = $videoFrame; } } $requests = getRequests($id, $testPath, $run, $cached, $secure, $haveLocations, false, true); $ret['domains'] = getDomainBreakdown($id, $testPath, $run, $cached, $requests); $ret['breakdown'] = getBreakdown($id, $testPath, $run, $cached, $requests); // check if removing requests $addRequests = 1; if (isset($_GET['requests'])) { if ($_GET['requests'] == 0) { $addRequests = 0; } } // add requests if ($addRequests == 1) { $ret['requests'] = $requests; } $console_log = DevToolsGetConsoleLog($testPath, $run, $cached); if (isset($console_log)) { $ret['consoleLog'] = $console_log; } if (gz_is_file("{$testPath}/{$run}{$cachedText}_status.txt")) { $ret['status'] = array(); $lines = gz_file("{$testPath}/{$run}{$cachedText}_status.txt"); foreach ($lines as $line) { $line = trim($line); if (strlen($line)) { list($time, $message) = explode("\t", $line); if (strlen($time) && strlen($message)) { $ret['status'][] = array('time' => $time, 'message' => $message); } } } } } } return $ret; }
/** * Do any aggregation once all of the tests have finished * * @param mixed $state */ function CollectResults(&$test, &$data) { global $logFile; $testPath = './' . GetTestPath($test['id']); logMsg("Loading page data from {$testPath}", "./log/{$logFile}", true); $page_data = loadAllPageData($testPath); if (count($page_data)) { foreach ($page_data as $run => &$page_run) { foreach ($page_run as $cached => &$test_data) { $data_row = $test_data; unset($data_row['URL']); // figure out the per-type request info (todo: measure how expensive this is and see if we have a better way) $breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests); foreach ($breakdown as $mime => &$values) { $data_row["{$mime}_requests"] = $values['requests']; $data_row["{$mime}_bytes"] = $values['bytes']; } // capture the page speed score if ($cached) { $data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt"); } else { $data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt"); } $data_row['url'] = $test['url']; $data_row['label'] = $test['label']; $data_row['location'] = $test['location']; $data_row['config'] = $test['config']; $data_row['cached'] = $cached; $data_row['run'] = $run; $data_row['id'] = $test['id']; $data[] = $data_row; $test['has_data'] = 1; } } } else { $data_row = array(); $data_row['url'] = $test['url']; $data_row['label'] = $test['label']; $data_row['location'] = $test['location']; $data_row['config'] = $test['config']; $data_row['id'] = $test['id']; $data[] = $data_row; } }
$tester = $test['testinfo']['tester']; } if (array_key_exists('test_runs', $test['testinfo']) && array_key_exists($i, $test['testinfo']['test_runs']) && array_key_exists('tester', $test['testinfo']['test_runs'][$i])) { $tester = $test['testinfo']['test_runs'][$i]['tester'] . '<br>'; } if (isset($tester)) { echo "<tester>" . xml_entities($tester) . "</tester>\n"; } } echo "<results>\n"; foreach ($pageData[$i][1] as $key => $val) { $key = preg_replace('/[^a-zA-Z0-9\\.\\-_]/', '_', $key); echo "<{$key}>" . xml_entities($val) . "</{$key}>\n"; } if ($pagespeed) { $score = GetPageSpeedScore("{$testPath}/{$i}_Cached_pagespeed.txt"); if (strlen($score)) { echo "<PageSpeedScore>{$score}</PageSpeedScore>\n"; } } echo "</results>\n"; // links to the relevant pages echo "<pages>\n"; echo "<details>http://{$host}{$uri}/result/{$id}/{$i}/details/cached/</details>\n"; echo "<checklist>http://{$host}{$uri}/result/{$id}/{$i}/performance_optimization/cached/</checklist>\n"; echo "<report>http://{$host}{$uri}/result/{$id}/{$i}/optimization_report/cached/</report>\n"; echo "<breakdown>http://{$host}{$uri}/result/{$id}/{$i}/breakdown/</breakdown>\n"; echo "<domains>http://{$host}{$uri}/result/{$id}/{$i}/domains/</domains>\n"; echo "<screenShot>http://{$host}{$uri}/result/{$id}/{$i}/screen_shot/cached/</screenShot>\n"; echo "</pages>\n"; // urls for the relevant images
function CollectTestResult($test, &$data) { global $benchmark; $id = $test['id']; $count = 0; echo "Reprocessing Test {$id}..."; logMsg("Reprocessing Test {$id}", "./log/reprocess-{$benchmark}.log", true); RestoreTest($id); ReprocessVideo($id); $testPath = './' . GetTestPath($id); $page_data = loadAllPageData($testPath); if (count($page_data)) { foreach ($page_data as $run => &$page_run) { foreach ($page_run as $cached => &$test_data) { $data_row = $test_data; unset($data_row['URL']); // figure out the per-type request info (todo: measure how expensive this is and see if we have a better way) $breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests); foreach ($breakdown as $mime => &$values) { $data_row["{$mime}_requests"] = $values['requests']; $data_row["{$mime}_bytes"] = $values['bytes']; } // capture the page speed score if ($cached) { $data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt"); } else { $data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt"); } $data_row['url'] = $test['url']; $data_row['label'] = $test['label']; $data_row['location'] = $test['location']; $data_row['config'] = $test['config']; $data_row['cached'] = $cached; $data_row['run'] = $run; $data_row['id'] = $test['id']; $data[] = $data_row; $count++; } } } else { $data_row = array(); $data_row['url'] = $test['url']; $data_row['label'] = $test['label']; $data_row['location'] = $test['location']; $data_row['config'] = $test['config']; $data_row['id'] = $test['id']; $data[] = $data_row; } // If the test was already archived, re-archive it. $testInfo = GetTestInfo($id); if (array_key_exists('archived', $testInfo) && $testInfo['archived']) { $lock = LockTest($id); if (isset($lock)) { $testInfo = GetTestInfo($id); $testInfo['archived'] = false; SaveTestInfo($id, $testInfo); UnlockTest($lock); } ArchiveTest($id); } echo "{$count} results\n"; }
/** * Build the data set * * @param mixed $pageData */ function BuildHAR(&$pageData, $id, $testPath, $options) { $result = array(); $entries = array(); $result['log'] = array(); $result['log']['version'] = '1.1'; $result['log']['creator'] = array('name' => 'WebPagetest', 'version' => VER_WEBPAGETEST); $result['log']['pages'] = array(); foreach ($pageData as $run => $pageRun) { foreach ($pageRun as $cached => $data) { $cached_text = ''; if ($cached) { $cached_text = '_Cached'; } if (!array_key_exists('browser', $result['log'])) { $result['log']['browser'] = array('name' => $data['browser_name'], 'version' => $data['browser_version']); } $pd = array(); $pd['startedDateTime'] = msdate($data['date']); $pd['title'] = "Run {$run}, "; if ($cached) { $pd['title'] .= "Repeat View"; } else { $pd['title'] .= "First View"; } $pd['title'] .= " for " . $data['URL']; $pd['id'] = "page_{$run}_{$cached}"; $pd['pageTimings'] = array('onLoad' => $data['docTime'], 'onContentLoad' => -1, '_startRender' => $data['render']); // add the pagespeed score if (gz_is_file("{$testPath}/{$run}{$cached_text}_pagespeed.txt")) { $pagespeed_data = LoadPageSpeedData("{$testPath}/{$run}{$cached_text}_pagespeed.txt"); if ($pagespeed_data) { $score = GetPageSpeedScore(null, $pagespeed_data); if (strlen($score)) { $pd['_pageSpeed'] = array('score' => $score, 'result' => $pagespeed_data); } } } // dump all of our metrics into the har data as custom fields foreach ($data as $name => $value) { if (!is_array($value)) { $pd["_{$name}"] = $value; } } // add the page-level ldata to the result $result['log']['pages'][] = $pd; // now add the object-level data to the result $secure = false; $haveLocations = false; $requests = getRequests($id, $testPath, $run, $cached, $secure, $haveLocations, false, true); foreach ($requests as &$r) { $entry = array(); $entry['pageref'] = $pd['id']; $entry['startedDateTime'] = msdate((double) $data['date'] + $r['load_start'] / 1000.0); $entry['time'] = $r['all_ms']; $request = array(); $request['method'] = $r['method']; $protocol = $r['is_secure'] ? 'https://' : 'http://'; $request['url'] = $protocol . $r['host'] . $r['url']; $request['headersSize'] = -1; $request['bodySize'] = -1; $request['cookies'] = array(); $request['headers'] = array(); $ver = ''; $headersSize = 0; if (isset($r['headers']) && isset($r['headers']['request'])) { foreach ($r['headers']['request'] as &$header) { $headersSize += strlen($header) + 2; // add 2 for the \r\n that is on the raw headers $pos = strpos($header, ':'); if ($pos > 0) { $name = trim(substr($header, 0, $pos)); $val = trim(substr($header, $pos + 1)); if (strlen($name)) { $request['headers'][] = array('name' => $name, 'value' => $val); } // parse out any cookies if (!strcasecmp($name, 'cookie')) { $cookies = explode(';', $val); foreach ($cookies as &$cookie) { $pos = strpos($cookie, '='); if ($pos > 0) { $name = (string) trim(substr($cookie, 0, $pos)); $val = (string) trim(substr($cookie, $pos + 1)); if (strlen($name)) { $request['cookies'][] = array('name' => $name, 'value' => $val); } } } } } else { $pos = strpos($header, 'HTTP/'); if ($pos >= 0) { $ver = (string) trim(substr($header, $pos + 5, 3)); } } } } if ($headersSize) { $request['headersSize'] = $headersSize; } $request['httpVersion'] = $ver; $request['queryString'] = array(); $parts = parse_url($request['url']); if (isset($parts['query'])) { $qs = array(); parse_str($parts['query'], $qs); foreach ($qs as $name => $val) { $request['queryString'][] = array('name' => (string) $name, 'value' => (string) $val); } } if (!strcasecmp(trim($request['method']), 'post')) { $request['postData'] = array(); $request['postData']['mimeType'] = ''; $request['postData']['text'] = ''; } $entry['request'] = $request; $response = array(); $response['status'] = (int) $r['responseCode']; $response['statusText'] = ''; $response['headersSize'] = -1; $response['bodySize'] = (int) $r['objectSize']; $response['headers'] = array(); $ver = ''; $loc = ''; $headersSize = 0; if (isset($r['headers']) && isset($r['headers']['response'])) { foreach ($r['headers']['response'] as &$header) { $headersSize += strlen($header) + 2; // add 2 for the \r\n that is on the raw headers $pos = strpos($header, ':'); if ($pos > 0) { $name = (string) trim(substr($header, 0, $pos)); $val = (string) trim(substr($header, $pos + 1)); if (strlen($name)) { $response['headers'][] = array('name' => $name, 'value' => $val); } if (!strcasecmp($name, 'location')) { $loc = (string) $val; } } else { $pos = strpos($header, 'HTTP/'); if ($pos >= 0) { $ver = (string) trim(substr($header, $pos + 5, 3)); } } } } if ($headersSize) { $response['headersSize'] = $headersSize; } $response['httpVersion'] = $ver; $response['redirectURL'] = $loc; $response['content'] = array(); $response['content']['size'] = (int) $r['objectSize']; if (isset($r['contentType']) && strlen($r['contentType'])) { $response['content']['mimeType'] = (string) $r['contentType']; } else { $response['content']['mimeType'] = ''; } // unsupported fields that are required $response['cookies'] = array(); $entry['response'] = $response; $entry['cache'] = (object) array(); $timings = array(); $timings['blocked'] = -1; $timings['dns'] = (int) $r['dns_ms']; if (!$timings['dns']) { $timings['dns'] = -1; } // HAR did not have an ssl time until version 1.2 . For // backward compatibility, "connect" includes "ssl" time. // WepbageTest's internal representation does not assume any // overlap, so we must add our connect and ssl time to get the // connect time expected by HAR. $timings['connect'] = durationOfInterval($r['connect_ms']) + durationOfInterval($r['ssl_ms']); if (!$timings['connect']) { $timings['connect'] = -1; } $timings['ssl'] = (int) $r['ssl_ms']; if (!$timings['ssl']) { $timings['ssl'] = -1; } // TODO(skerner): WebpageTest's data model has no way to // represent the difference between the states HAR calls // send (time required to send HTTP request to the server) // and wait (time spent waiting for a response from the server). // We lump both into "wait". Issue 24* tracks this work. When // it is resolved, read the real values for send and wait // instead of using the request's TTFB. // *: http://code.google.com/p/webpagetest/issues/detail?id=24 $timings['send'] = 0; $timings['wait'] = (int) $r['ttfb_ms']; $timings['receive'] = (int) $r['download_ms']; $entry['timings'] = $timings; // The HAR spec defines time as the sum of the times in the // timings object, excluding any unknown (-1) values and ssl // time (which is included in "connect", for backward // compatibility with tools written before "ssl" was defined // in HAR version 1.2). $entry['time'] = 0; foreach ($timings as $timingKey => $duration) { if ($timingKey != 'ssl' && $duration != UNKNOWN_TIME) { $entry['time'] += $duration; } } if (array_key_exists('custom_rules', $r)) { $entry['_custom_rules'] = $r['custom_rules']; } // dump all of our metrics into the har data as custom fields foreach ($r as $name => $value) { if (!is_array($value)) { $entry["_{$name}"] = $value; } } // add it to the list of entries $entries[] = $entry; } // add the bodies to the requests if (isset($options['bodies']) && $options['bodies']) { $bodies_file = $testPath . '/' . $run . $cached_text . '_bodies.zip'; if (is_file($bodies_file)) { $zip = new ZipArchive(); if ($zip->open($bodies_file) === TRUE) { for ($i = 0; $i < $zip->numFiles; $i++) { $index = intval($zip->getNameIndex($i), 10) - 1; if (array_key_exists($index, $entries)) { $entries[$index]['response']['content']['text'] = utf8_encode($zip->getFromIndex($i)); } } } } } } } $result['log']['entries'] = $entries; return $result; }
/** * @return string The score */ public function getPageSpeedScore() { // TODO: move implementation to this method if ($this->fileHandler->gzFileExists($this->localPaths->pageSpeedFile())) { return GetPageSpeedScore($this->localPaths->pageSpeedFile()); } return null; }