function importPageMod($hPage)
{
global $pagesTable, $requestsTable;
$t_CVSNO = time();
$pageid = $hPage['pageid'];
$wptid = $hPage['wptid'];
$wptrun = $hPage['wptrun'];
if (!$wptid || !$wptrun) {
tprint("ERROR: importPageMod({$pageid}): failed to find wptid and wptrun: {$wptid}, {$wptrun}");
return;
}
// lifted from importWptResults
$wptServer = wptServer();
$request = $wptServer . "export.php?test={$wptid}&run={$wptrun}&cached=0&php=1";
$response = fetchUrl($request);
//tprint("after fetchUrl", $t_CVSNO);
if (!strlen($response)) {
tprint("ERROR: importPageMod({$pageid}): URL failed: {$request}");
return;
}
// lifted from importHarJson
$json_text = $response;
$HAR = json_decode($json_text);
if (NULL == $HAR) {
tprint("ERROR: importPageMod({$pageid}): JSON decode failed");
return;
}
$log = $HAR->{'log'};
$pages = $log->{'pages'};
$pagecount = count($pages);
if (0 == $pagecount) {
tprint("ERROR: importPageMod({$pageid}): No pages found");
return;
}
// lifted from importPage
$page = $pages[0];
if (array_key_exists('_TTFB', $page)) {
$hPage['TTFB'] = $page->{'_TTFB'};
}
if (array_key_exists('_fullyLoaded', $page)) {
$hPage['fullyLoaded'] = $page->{'_fullyLoaded'};
}
if (array_key_exists('_visualComplete', $page)) {
$hPage['visualComplete'] = $page->{'_visualComplete'};
}
if (array_key_exists('_gzip_total', $page)) {
$hPage['gzipTotal'] = $page->{'_gzip_total'};
$hPage['gzipSavings'] = $page->{'_gzip_savings'};
}
if (array_key_exists('_domElements', $page)) {
$hPage['numDomElements'] = $page->{'_domElements'};
}
if (array_key_exists('_domContentLoadedEventStart', $page)) {
$hPage['onContentLoaded'] = $page->{'_domContentLoadedEventStart'};
}
if (array_key_exists('_base_page_cdn', $page)) {
$hPage['cdn'] = $page->{'_base_page_cdn'};
}
if (array_key_exists('_SpeedIndex', $page)) {
$hPage['SpeedIndex'] = $page->{'_SpeedIndex'};
}
// lifted from aggregateStats
// initialize variables for counting the page's stats
$hPage['bytesTotal'] = 0;
$hPage['reqTotal'] = 0;
$typeMap = array("flash" => "Flash", "css" => "CSS", "image" => "Img", "script" => "JS", "html" => "Html", "font" => "Font", "other" => "Other", "gif" => "Gif", "jpg" => "Jpg", "png" => "Png");
foreach (array_keys($typeMap) as $type) {
// initialize the hashes
$hPage['req' . $typeMap[$type]] = 0;
$hPage['bytes' . $typeMap[$type]] = 0;
}
$hDomains = array();
$hPage['maxageNull'] = $hPage['maxage0'] = $hPage['maxage1'] = $hPage['maxage30'] = $hPage['maxage365'] = $hPage['maxageMore'] = 0;
$hPage['bytesHtmlDoc'] = $hPage['numRedirects'] = $hPage['numErrors'] = $hPage['numGlibs'] = $hPage['numHttps'] = $hPage['numCompressed'] = $hPage['maxDomainReqs'] = 0;
$result = doQuery("select mimeType, urlShort, resp_content_type, respSize, expAge, firstHtml, status, resp_content_encoding, req_host from {$requestsTable} where pageid = {$pageid};");
//tprint("after query", $t_CVSNO);
while ($row = mysql_fetch_assoc($result)) {
$reqUrl = $row['urlShort'];
$mimeType = prettyType($row['mimeType'], $reqUrl);
$respSize = intval($row['respSize']);
$hPage['reqTotal']++;
$hPage['bytesTotal'] += $respSize;
$hPage['req' . $typeMap[$mimeType]]++;
$hPage['bytes' . $typeMap[$mimeType]] += $respSize;
if ("image" === $mimeType) {
$content_type = $row['resp_content_type'];
$imgformat = false !== stripos($content_type, "image/gif") ? "gif" : (false !== stripos($content_type, "image/jpg") || false !== stripos($content_type, "image/jpeg") ? "jpg" : (false !== stripos($content_type, "image/png") ? "png" : ""));
if ($imgformat) {
$hPage['req' . $typeMap[$imgformat]]++;
$hPage['bytes' . $typeMap[$imgformat]] += $respSize;
}
}
// count unique domains (really hostnames)
$aMatches = array();
if ($reqUrl && preg_match('/http[s]*:\\/\\/([^\\/]*)/', $reqUrl, $aMatches)) {
$hostname = $aMatches[1];
if (!array_key_exists($hostname, $hDomains)) {
$hDomains[$hostname] = 0;
}
$hDomains[$hostname]++;
// count hostnames
} else {
tprint("ERROR: importPageMod({$pageid}): No hostname found in URL: {$reqUrl}");
}
// count expiration windows
$expAge = $row['expAge'];
$daySecs = 24 * 60 * 60;
if (NULL === $expAge) {
$hPage['maxageNull']++;
} else {
if (0 === intval($expAge)) {
$hPage['maxage0']++;
} else {
if ($expAge <= 1 * $daySecs) {
$hPage['maxage1']++;
} else {
if ($expAge <= 30 * $daySecs) {
$hPage['maxage30']++;
} else {
if ($expAge <= 365 * $daySecs) {
$hPage['maxage365']++;
} else {
$hPage['maxageMore']++;
}
}
}
}
}
if ($row['firstHtml']) {
$hPage['bytesHtmlDoc'] = $respSize;
}
// CVSNO - can we get this UNgzipped?!
$status = $row['status'];
if (300 <= $status && $status < 400 && 304 != $status) {
$hPage['numRedirects']++;
} else {
if (400 <= $status && $status < 600) {
$hPage['numErrors']++;
}
}
if (0 === stripos($reqUrl, "https://")) {
$hPage['numHttps']++;
}
if (FALSE !== stripos($row['req_host'], "googleapis.com")) {
$hPage['numGlibs']++;
}
if ("gzip" == $row['resp_content_encoding'] || "deflate" == $row['resp_content_encoding']) {
$hPage['numCompressed']++;
}
}
mysql_free_result($result);
$hPage['numDomains'] = count(array_keys($hDomains));
foreach (array_keys($hDomains) as $domain) {
$hPage['maxDomainReqs'] = max($hPage['maxDomainReqs'], $hDomains[$domain]);
}
//$cmd = "UPDATE $pagesTable SET reqTotal = $reqTotal, bytesTotal = $bytesTotal" .
$cmd = "insert into pagestmp SET " . hashImplode(", ", "=", $hPage) . ";";
//tprint("before insert", $t_CVSNO);
doSimpleCommand($cmd);
//tprint("after insert\n", $t_CVSNO);
}
} else {
// start this task
lprint("Starting task \"{$task}\"...");
// fork the child process
// return:
// -1 - error
// 0 - we're the forked child process
// >0 - we're the parent process and this is the process ID of the child process
$pid = pcntl_fork();
if (-1 === $pid) {
die("ERROR: failed to fork child process.\n");
}
if ($pid) {
// parent process - save the child process ID
$aChildPids[] = $pid;
tprint("{$task} pid = {$pid}");
} else {
// child process
if ("submit" === $task) {
// Submit the jobs to WebPagetest.
submitBatch();
} else {
if ("status" === $task) {
// Check the test status with WPT server
checkWPTStatus();
} else {
if ("obtain" === $task) {
// Obtain XML result
obtainXMLResults();
} else {
if (0 === strpos($task, "parse")) {