/**
* Re-process all of the video for an existing test
*
* @param mixed $id
*/
function ReprocessVideo($id)
{
$testPath = './' . GetTestPath($id);
if (is_dir($testPath)) {
$lock = LockTest($id);
if (isset($lock)) {
$cacheFiles = glob("{$testPath}/*.dat.gz");
if ($cacheFiles && is_array($cacheFiles) && count($cacheFiles)) {
foreach ($cacheFiles as $cacheFile) {
unlink($cacheFile);
}
}
$videoFiles = glob("{$testPath}/*.mp4");
if ($videoFiles && is_array($videoFiles) && count($videoFiles)) {
foreach ($videoFiles as $video) {
if (preg_match('/^.*\\/(?P<run>[0-9]+)(?P<cached>_Cached)?_video\\.mp4$/i', $video, $matches)) {
$run = $matches['run'];
$cached = array_key_exists('cached', $matches) ? 1 : 0;
$videoDir = "{$testPath}/video_{$run}";
if ($cached) {
$videoDir .= '_cached';
}
delTree($videoDir, false);
ProcessAVIVideo($id, $testPath, $run, $cached);
}
}
}
UnlockTest($lock);
}
}
}
/**
* Cancel and individual test
*
* @param mixed $id
*/
function CancelTest($id)
{
$lock = LockTest($id);
if ($lock) {
$cancelled = false;
$testInfo = GetTestInfo($id);
if ($testInfo && !array_key_exists('started', $testInfo)) {
$testInfo['cancelled'] = time();
SaveTestInfo($id, $testInfo);
// delete the actual test file.
if (array_key_exists('workdir', $testInfo)) {
$ext = 'url';
if ($testInfo['priority']) {
$ext = "p{$testInfo['priority']}";
}
$queued_job_file = $testInfo['workdir'] . "/{$id}.{$ext}";
$cancelled = unlink($queued_job_file);
}
}
UnlockTest($lock);
}
return $cancelled;
}
}
$update = array();
$update['id'] = $id;
$update['last_updated'] = $now;
$ind[$ini['industry']][$ini['industry_page']] = $update;
$data = json_encode($ind);
file_put_contents('./video/dat/industry.dat', $data);
Unlock($indLock);
}
}
}
if ($testInfo_dirty) {
SaveTestInfo($id, $testInfo);
}
SecureDir($testPath);
UnlockTest($testLock);
/*************************************************************************
* Do No modify TestInfo after this point
**************************************************************************/
// do any post-processing when the full test is complete that doesn't rely on testinfo
if ($done) {
logTestMsg($id, "Test Complete");
// send an async request to the post-processing code so we don't block
SendAsyncRequest("/work/postprocess.php?test={$id}");
}
} else {
logMsg("location key incorrect\n");
}
}
}
}
/**
* Get an actual task to complete
*
*/
function GetJob()
{
$is_done = false;
global $location;
global $key;
global $pc;
global $ec2;
global $tester;
global $recover;
global $is_json;
global $dnsServers;
$workDir = "./work/jobs/{$location}";
$locKey = GetLocationKey($location);
if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) {
if ($lock = LockLocation($location)) {
$now = time();
$testers = GetTesters($location, true);
// make sure the tester isn't marked as offline (usually when shutting down EC2 instances)
$testerCount = isset($testers['testers']) ? count($testers['testers']) : 0;
$testerIndex = null;
$offline = false;
if ($testerCount) {
if (strlen($ec2)) {
foreach ($testers['testers'] as $index => $testerInfo) {
if (isset($testerInfo['ec2']) && $testerInfo['ec2'] == $ec2 && isset($testerInfo['offline']) && $testerInfo['offline']) {
$offline = true;
}
break;
}
}
foreach ($testers['testers'] as $index => $testerInfo) {
if ($testerInfo['id'] == $tester) {
$testerIndex = $index;
break;
}
}
}
if (!$offline) {
$fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount);
if (isset($fileName) && strlen($fileName)) {
$is_done = true;
$delete = true;
if ($is_json) {
header("Content-type: application/json");
} else {
header('Content-type: text/plain');
}
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
// send the test info to the test agent
$testInfo = file_get_contents("{$workDir}/{$fileName}");
// extract the test ID from the job file
if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) {
$testId = trim($matches[1]);
}
if (isset($testId)) {
// figure out the path to the results
$testPath = './' . GetTestPath($testId);
// flag the test with the start time
$ini = file_get_contents("{$testPath}/testinfo.ini");
if (stripos($ini, 'startTime=') === false) {
$time = time();
$start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time);
$out = str_replace('[test]', $start, $ini);
file_put_contents("{$testPath}/testinfo.ini", $out);
}
$lock = LockTest($testId);
if ($lock) {
$testInfoJson = GetTestInfo($testId);
if ($testInfoJson) {
if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) {
$testInfoJson['tester'] = $tester;
}
if (isset($dnsServers) && strlen($dnsServers)) {
$testInfoJson['testerDNS'] = $dnsServers;
}
if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) {
$testInfoJson['started'] = $time;
logTestMsg($testId, "Starting test (initiated by tester {$tester})");
}
if (!array_key_exists('test_runs', $testInfoJson)) {
$testInfoJson['test_runs'] = array();
}
for ($run = 1; $run <= $testInfoJson['runs']; $run++) {
if (!array_key_exists($run, $testInfoJson['test_runs'])) {
$testInfoJson['test_runs'][$run] = array('done' => false);
}
}
$testInfoJson['id'] = $testId;
ProcessTestShard($testInfoJson, $testInfo, $delete);
SaveTestInfo($testId, $testInfoJson);
}
UnlockTest($lock);
}
file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId);
}
if ($delete) {
unlink("{$workDir}/{$fileName}");
} else {
AddJobFileHead($workDir, $fileName, $priority, true);
}
if ($is_json) {
$testJson = array();
$script = '';
$isScript = false;
$lines = explode("\r\n", $testInfo);
foreach ($lines as $line) {
if (strlen(trim($line))) {
if ($isScript) {
if (strlen($script)) {
$script .= "\r\n";
}
$script .= $line;
} elseif (!strcasecmp($line, '[Script]')) {
$isScript = true;
} else {
$pos = strpos($line, '=');
if ($pos !== false) {
$key = trim(substr($line, 0, $pos));
$value = trim(substr($line, $pos + 1));
if (strlen($key) && strlen($value)) {
if ($key == 'customMetric') {
$pos = strpos($value, ':');
if ($pos !== false) {
$metric = trim(substr($value, 0, $pos));
$code = base64_decode(substr($value, $pos + 1));
if ($code !== false && strlen($metric) && strlen($code)) {
if (!isset($testJson['customMetrics'])) {
$testJson['customMetrics'] = array();
}
$testJson['customMetrics'][$metric] = $code;
}
}
} elseif (is_numeric($value)) {
$testJson[$key] = (int) $value;
} else {
$testJson[$key] = $value;
}
}
}
}
}
}
if (strlen($script)) {
$testJson['script'] = $script;
}
echo json_encode($testJson);
} else {
echo $testInfo;
}
$ok = true;
}
// zero out the tracked page loads in case some got lost
if (!$is_done && is_file("./tmp/{$location}.tests")) {
$tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true);
if ($tests) {
$tests['tests'] = 0;
file_put_contents("./tmp/{$location}.tests", json_encode($tests));
}
}
}
UnlockLocation($lock);
// keep track of the last time this location reported in
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
$testerInfo['pc'] = $pc;
$testerInfo['ec2'] = $ec2;
$testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver'];
$testerInfo['freedisk'] = @$_GET['freedisk'];
$testerInfo['ie'] = @$_GET['ie'];
$testerInfo['dns'] = $dnsServers;
$testerInfo['video'] = @$_GET['video'];
$testerInfo['GPU'] = @$_GET['GPU'];
$testerInfo['test'] = '';
if (isset($testId)) {
$testerInfo['test'] = $testId;
}
UpdateTester($location, $tester, $testerInfo);
}
}
return $is_done;
}
if (!$list) {
unset($id);
}
// make sure there are no risky files and that nothing is allowed execute permission
SecureDir($path);
// mark the test as piblished so we won't expose a resubmit
$lock = LockTest($id);
if ($lock) {
$testInfo = GetTestInfo($id);
if ($testInfo) {
$testInfo['id'] = $id;
$testInfo['job'] = $id;
$testInfo['published'] = true;
if (array_key_exists('noscript', $_REQUEST) && $_REQUEST['noscript']) {
$testInfo['script'] = null;
}
SaveTestInfo($id, $testInfo);
}
UnlockTest($lock);
}
if (is_file("{$path}/testinfo.ini")) {
$ini = file("{$path}/testinfo.ini");
foreach ($ini as &$line) {
if (!strncasecmp($line, 'id=', 3)) {
$line = "id={$id}\r\n";
}
}
file_put_contents("{$path}/testinfo.ini", implode('', $ini));
}
echo $id;
}
/**
* Get an actual task to complete
*
*/
function GetJob()
{
$is_done = false;
global $location;
global $key;
global $pc;
global $ec2;
global $tester;
global $recover;
global $is_json;
global $dnsServers;
global $screenwidth;
global $screenheight;
global $winver;
global $isWinServer;
global $isWin64;
$workDir = "./work/jobs/{$location}";
$locInfo = GetLocationInfo($location);
$locKey = '';
if (isset($locInfo) && is_array($locInfo) && isset($locInfo['key'])) {
$locKey = $locInfo['key'];
}
$incremental = true;
if (GetSetting('no_incremental') || isset($locInfo['incremental']) && !$locInfo['incremental']) {
$incremental = false;
}
if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) {
$now = time();
// If it is an EC2 auto-scaling location, make sure the agent isn't marked as offline
$offline = false;
if (isset($locInfo['ami'])) {
$testers = GetTesters($location, true);
// make sure the tester isn't marked as offline (usually when shutting down EC2 instances)
$testerCount = isset($testers['testers']) ? count($testers['testers']) : 0;
$testerIndex = null;
if ($testerCount) {
if (strlen($ec2)) {
foreach ($testers['testers'] as $index => $testerInfo) {
if (isset($testerInfo['ec2']) && $testerInfo['ec2'] == $ec2 && isset($testerInfo['offline']) && $testerInfo['offline']) {
$offline = true;
}
break;
}
}
foreach ($testers['testers'] as $index => $testerInfo) {
if ($testerInfo['id'] == $tester) {
$testerIndex = $index;
break;
}
}
}
}
if (!$offline) {
$testInfo = GetTestJob($location, $fileName, $workDir, $priority, $testInfo, $pc, $testerIndex, $testerCount);
if (isset($testInfo)) {
$original_test_info = $testInfo;
$is_done = true;
$delete = true;
if ($is_json) {
header("Content-type: application/json");
} else {
header('Content-type: text/plain');
}
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
// send the test info to the test agent
$newline = strpos($testInfo, "\n", 2);
if ($newline) {
$newline++;
$after = substr($testInfo, $newline);
$testInfo = substr($testInfo, 0, $newline);
$software = GetSetting('software');
if ($software) {
$testInfo .= "software={$software}\r\n";
}
if (GetSetting('enable_agent_processing')) {
$testInfo .= "processResults=1\r\n";
}
if (!$incremental) {
$testInfo .= "incremental=0\r\n";
}
$testInfo .= $after;
}
// extract the test ID from the job file
if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) {
$testId = trim($matches[1]);
}
if (isset($testId)) {
// figure out the path to the results
$testPath = './' . GetTestPath($testId);
// flag the test with the start time
$ini = file_get_contents("{$testPath}/testinfo.ini");
if (stripos($ini, 'startTime=') === false) {
$time = time();
$start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time);
$out = str_replace('[test]', $start, $ini);
file_put_contents("{$testPath}/testinfo.ini", $out);
}
$lock = LockTest($testId);
if ($lock) {
$testInfoJson = GetTestInfo($testId);
if ($testInfoJson) {
if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) {
$testInfoJson['tester'] = $tester;
}
if (isset($dnsServers) && strlen($dnsServers)) {
$testInfoJson['testerDNS'] = $dnsServers;
}
if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) {
$testInfoJson['started'] = $time;
logTestMsg($testId, "Starting test (initiated by tester {$tester})");
}
if (!array_key_exists('test_runs', $testInfoJson)) {
$testInfoJson['test_runs'] = array();
}
for ($run = 1; $run <= $testInfoJson['runs']; $run++) {
if (!array_key_exists($run, $testInfoJson['test_runs'])) {
$testInfoJson['test_runs'][$run] = array('done' => false);
}
}
$dotPos = stripos($testId, ".");
$testInfoJson['id'] = $dotPos === false ? $testId : substr($testId, $dotPos + 1);
ProcessTestShard($testInfoJson, $testInfo, $delete, $priority);
SaveTestInfo($testId, $testInfoJson);
}
UnlockTest($lock);
}
}
if ($delete) {
if (isset($fileName) && is_file("{$workDir}/{$fileName}")) {
unlink("{$workDir}/{$fileName}");
}
} else {
AddTestJobHead($location, $original_test_info, $workDir, $fileName, $priority, true);
}
if ($is_json) {
$testJson = array();
$script = '';
$isScript = false;
$lines = explode("\r\n", $testInfo);
foreach ($lines as $line) {
if (strlen(trim($line))) {
if ($isScript) {
if (strlen($script)) {
$script .= "\r\n";
}
$script .= $line;
} elseif (!strcasecmp($line, '[Script]')) {
$isScript = true;
} else {
$pos = strpos($line, '=');
if ($pos !== false) {
$key = trim(substr($line, 0, $pos));
$value = trim(substr($line, $pos + 1));
if (strlen($key) && strlen($value)) {
if ($key == 'customMetric') {
$pos = strpos($value, ':');
if ($pos !== false) {
$metric = trim(substr($value, 0, $pos));
$code = base64_decode(substr($value, $pos + 1));
if ($code !== false && strlen($metric) && strlen($code)) {
if (!isset($testJson['customMetrics'])) {
$testJson['customMetrics'] = array();
}
$testJson['customMetrics'][$metric] = $code;
}
}
} elseif (is_numeric($value)) {
$testJson[$key] = (int) $value;
} else {
$testJson[$key] = $value;
}
}
}
}
}
}
if (strlen($script)) {
$testJson['script'] = $script;
}
// See if we need to include apk information
if (isset($_REQUEST['apk']) && is_file(__DIR__ . '/update/apk.dat')) {
$apk_info = json_decode(file_get_contents(__DIR__ . '/update/apk.dat'), true);
if (isset($apk_info) && is_array($apk_info) && isset($apk_info['packages']) && is_array($apk_info['packages'])) {
$protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on' || isset($_SERVER['HTTP_SSL']) && $_SERVER['HTTP_SSL'] == 'On' ? 'https' : 'http';
$update_path = dirname($_SERVER['PHP_SELF']) . '/update/';
$base_uri = "{$protocol}://{$_SERVER['HTTP_HOST']}{$update_path}";
foreach ($apk_info['packages'] as $package => $info) {
$apk_info['packages'][$package]['apk_url'] = "{$base_uri}{$apk_info['packages'][$package]['file_name']}?md5={$apk_info['packages'][$package]['md5']}";
}
$testJson['apk_info'] = $apk_info;
}
}
echo json_encode($testJson);
} else {
echo $testInfo;
}
$ok = true;
}
// keep track of the last time this location reported in
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
$testerInfo['pc'] = $pc;
$testerInfo['ec2'] = $ec2;
$testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver'];
$testerInfo['freedisk'] = @$_GET['freedisk'];
$testerInfo['ie'] = @$_GET['ie'];
$testerInfo['dns'] = $dnsServers;
$testerInfo['video'] = @$_GET['video'];
$testerInfo['GPU'] = @$_GET['GPU'];
$testerInfo['screenwidth'] = $screenwidth;
$testerInfo['screenheight'] = $screenheight;
$testerInfo['winver'] = $winver;
$testerInfo['isWinServer'] = $isWinServer;
$testerInfo['isWin64'] = $isWin64;
$testerInfo['test'] = '';
if (isset($testId)) {
$testerInfo['test'] = $testId;
}
UpdateTester($location, $tester, $testerInfo);
}
}
return $is_done;
}
function CollectTestResult($test, &$data)
{
global $benchmark;
$id = $test['id'];
$count = 0;
echo "Reprocessing Test {$id}...";
logMsg("Reprocessing Test {$id}", "./log/reprocess-{$benchmark}.log", true);
RestoreTest($id);
ReprocessVideo($id);
$testPath = './' . GetTestPath($id);
$page_data = loadAllPageData($testPath);
if (count($page_data)) {
foreach ($page_data as $run => &$page_run) {
foreach ($page_run as $cached => &$test_data) {
$data_row = $test_data;
unset($data_row['URL']);
// figure out the per-type request info (todo: measure how expensive this is and see if we have a better way)
$breakdown = getBreakdown($test['id'], $testPath, $run, $cached, $requests);
foreach ($breakdown as $mime => &$values) {
$data_row["{$mime}_requests"] = $values['requests'];
$data_row["{$mime}_bytes"] = $values['bytes'];
}
// capture the page speed score
if ($cached) {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_Cached_pagespeed.txt");
} else {
$data_row['page_speed'] = GetPageSpeedScore("{$testPath}/{$run}_pagespeed.txt");
}
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['cached'] = $cached;
$data_row['run'] = $run;
$data_row['id'] = $test['id'];
$data[] = $data_row;
$count++;
}
}
} else {
$data_row = array();
$data_row['url'] = $test['url'];
$data_row['label'] = $test['label'];
$data_row['location'] = $test['location'];
$data_row['config'] = $test['config'];
$data_row['id'] = $test['id'];
$data[] = $data_row;
}
// If the test was already archived, re-archive it.
$testInfo = GetTestInfo($id);
if (array_key_exists('archived', $testInfo) && $testInfo['archived']) {
$lock = LockTest($id);
if (isset($lock)) {
$testInfo = GetTestInfo($id);
$testInfo['archived'] = false;
SaveTestInfo($id, $testInfo);
UnlockTest($lock);
}
ArchiveTest($id);
}
echo "{$count} results\n";
}
