$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
if (!isset($testerError)) {
$testerError = false;
}
if (array_key_exists('testerror', $_REQUEST) && strlen($_REQUEST['testerror'])) {
$testerError = $_REQUEST['testerror'];
} elseif (array_key_exists('error', $_REQUEST) && strlen($_REQUEST['error'])) {
$testerError = $_REQUEST['error'];
}
// clear the rebooted flag on the first successful test
$rebooted = null;
if ($testerError === false) {
$rebooted = false;
}
UpdateTester($location, $tester, $testerInfo, $cpu, $testerError, $rebooted);
}
// see if the test is complete
if ($done) {
// Mark the test as done and save it out so that we can load the page data
$testInfo['completed'] = $time;
if (!array_key_exists('test_runs', $testInfo)) {
$testInfo['test_runs'] = array();
}
for ($run = 1; $run <= $testInfo['runs']; $run++) {
if (array_key_exists($run, $testInfo['test_runs'])) {
$testInfo['test_runs'][$run]['done'] = true;
} else {
$testInfo['test_runs'][$run] = array('done' => true);
}
}
/**
* Get an actual task to complete
*
*/
function GetJob()
{
$is_done = false;
global $location;
global $key;
global $pc;
global $ec2;
global $tester;
global $recover;
global $is_json;
global $dnsServers;
$workDir = "./work/jobs/{$location}";
$locKey = GetLocationKey($location);
if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) {
if ($lock = LockLocation($location)) {
$now = time();
$testers = GetTesters($location);
// make sure the tester isn't marked as offline (usually when shutting down EC2 instances)
$testerCount = isset($testers['testers']) ? count($testers['testers']) : 0;
$testerIndex = null;
if ($testerCount) {
foreach ($testers['testers'] as $index => $testerInfo) {
if ($testerInfo['id'] == $tester) {
$testerIndex = $index;
break;
}
}
}
$fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount);
if (isset($fileName) && strlen($fileName)) {
$is_done = true;
$delete = true;
if ($is_json) {
header("Content-type: application/json");
} else {
header('Content-type: text/plain');
}
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
// send the test info to the test agent
$testInfo = file_get_contents("{$workDir}/{$fileName}");
// extract the test ID from the job file
if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) {
$testId = trim($matches[1]);
}
if (isset($testId)) {
// figure out the path to the results
$testPath = './' . GetTestPath($testId);
// flag the test with the start time
$ini = file_get_contents("{$testPath}/testinfo.ini");
if (stripos($ini, 'startTime=') === false) {
$time = time();
$start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time);
$out = str_replace('[test]', $start, $ini);
file_put_contents("{$testPath}/testinfo.ini", $out);
}
$lock = LockTest($testId);
if ($lock) {
$testInfoJson = GetTestInfo($testId);
if ($testInfoJson) {
if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) {
$testInfoJson['tester'] = $tester;
}
if (isset($dnsServers) && strlen($dnsServers)) {
$testInfoJson['testerDNS'] = $dnsServers;
}
if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) {
$testInfoJson['started'] = $time;
logTestMsg($testId, "Starting test (initiated by tester {$tester})");
}
if (!array_key_exists('test_runs', $testInfoJson)) {
$testInfoJson['test_runs'] = array();
}
for ($run = 1; $run <= $testInfoJson['runs']; $run++) {
if (!array_key_exists($run, $testInfoJson['test_runs'])) {
$testInfoJson['test_runs'][$run] = array('done' => false);
}
}
$testInfoJson['id'] = $testId;
ProcessTestShard($testInfoJson, $testInfo, $delete);
SaveTestInfo($testId, $testInfoJson);
}
UnlockTest($lock);
}
file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId);
}
if ($delete) {
unlink("{$workDir}/{$fileName}");
} else {
AddJobFileHead($workDir, $fileName, $priority, true);
}
if ($is_json) {
$testJson = array();
$script = '';
$isScript = false;
$lines = explode("\r\n", $testInfo);
foreach ($lines as $line) {
if (strlen(trim($line))) {
if ($isScript) {
if (strlen($script)) {
$script .= "\r\n";
}
$script .= $line;
} elseif (!strcasecmp($line, '[Script]')) {
$isScript = true;
} else {
$pos = strpos($line, '=');
if ($pos > -1) {
$key = trim(substr($line, 0, $pos));
$value = trim(substr($line, $pos + 1));
if (strlen($key) && strlen($value)) {
if (is_numeric($value)) {
$testJson[$key] = (int) $value;
} else {
$testJson[$key] = $value;
}
}
}
}
}
}
if (strlen($script)) {
$testJson['script'] = $script;
}
echo json_encode($testJson);
} else {
echo $testInfo;
}
$ok = true;
}
// zero out the tracked page loads in case some got lost
if (!$is_done && is_file("./tmp/{$location}.tests")) {
$tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true);
if ($tests) {
$tests['tests'] = 0;
file_put_contents("./tmp/{$location}.tests", json_encode($tests));
}
}
UnlockLocation($lock);
// keep track of the last time this location reported in
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
$testerInfo['pc'] = $pc;
$testerInfo['ec2'] = $ec2;
$testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver'];
$testerInfo['freedisk'] = @$_GET['freedisk'];
$testerInfo['ie'] = @$_GET['ie'];
$testerInfo['dns'] = $dnsServers;
$testerInfo['video'] = @$_GET['video'];
$testerInfo['GPU'] = @$_GET['GPU'];
$testerInfo['test'] = '';
if (isset($testId)) {
$testerInfo['test'] = $testId;
}
UpdateTester($location, $tester, $testerInfo);
}
}
return $is_done;
}
/**
* See if we need to reboot this tester
*
*/
function GetReboot()
{
global $location;
global $pc;
global $ec2;
global $tester;
$reboot = false;
$name = @strlen($ec2) ? $ec2 : $pc;
if (isset($name) && strlen($name) && isset($location) && strlen($location)) {
$rebootFile = "./work/jobs/{$location}/{$name}.reboot";
if (is_file($rebootFile)) {
unlink($rebootFile);
$reboot = true;
}
}
// If we have a 100% error rate for the current PC, send it a reboot
if (!$reboot) {
$testers = GetTesters($location);
foreach ($testers as $t) {
if ($t['id'] == $tester && !$rebooted) {
if ($t['errors'] >= 100) {
UpdateTester($location, $tester, null, null, null, true);
$reboot = true;
}
break;
}
}
}
if ($reboot) {
header('Content-type: text/plain');
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
echo "Reboot";
}
return $reboot;
}
/**
* See if we need to reboot this tester
*
*/
function GetReboot()
{
global $location;
global $pc;
global $ec2;
global $tester;
$reboot = false;
$name = @strlen($ec2) ? $ec2 : $pc;
if (isset($name) && strlen($name) && isset($location) && strlen($location)) {
$rebootFile = "./work/jobs/{$location}/{$name}.reboot";
if (is_file($rebootFile)) {
unlink($rebootFile);
$reboot = true;
}
}
// If we have a 100% error rate for the current PC, send it a reboot
if (!$reboot) {
$testers = GetTesters($location);
foreach ($testers as $t) {
if ($t['id'] == $tester && !$rebooted) {
if ($t['errors'] >= 100) {
UpdateTester($location, $tester, null, null, null, true);
$reboot = true;
}
break;
}
}
}
// If we sent down more than 3 updates sequentially, reboot the tester
if (!$reboot && function_exists('apc_fetch') && function_exists('apc_store')) {
$updateCount = apc_fetch("uc-{$location}-{$tester}");
if ($updateCount && $updateCount >= 3) {
$reboot = true;
$updateCount = 0;
apc_store("uc-{$location}-{$tester}", $updateCount, 3600);
}
}
if ($reboot) {
header('Content-type: text/plain');
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
echo "Reboot";
}
return $reboot;
}
if ($testInfo['video']) {
$workdone_video_end = microtime(true);
}
}
if (strlen($location) && strlen($tester)) {
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
if (!isset($testerError)) {
$testerError = false;
}
if (array_key_exists('testerror', $_REQUEST) && strlen($_REQUEST['testerror'])) {
$testerError = $_REQUEST['testerror'];
} elseif (array_key_exists('error', $_REQUEST) && strlen($_REQUEST['error'])) {
$testerError = $_REQUEST['error'];
}
UpdateTester($location, $tester, $testerInfo, $cpu, $testerError);
}
// see if the test is complete
if ($done) {
// Mark the test as done and save it out so that we can load the page data
$testInfo['completed'] = $time;
if (!array_key_exists('test_runs', $testInfo)) {
$testInfo['test_runs'] = array();
}
for ($run = 1; $run <= $testInfo['runs']; $run++) {
if (array_key_exists($run, $testInfo['test_runs'])) {
$testInfo['test_runs'][$run]['done'] = true;
} else {
$testInfo['test_runs'][$run] = array('done' => true);
}
}
mkdir('./tmp');
}
touch("./tmp/{$location}.tm");
}
// Figure out the path to the results.
$ini = parse_ini_file("{$testPath}/testinfo.ini");
$time = time();
$testInfo['last_updated'] = $time;
$testInfo_dirty = true;
if (!strlen($tester) && array_key_exists('tester', $testInfo) && strlen($testInfo['tester'])) {
$tester = $testInfo['tester'];
}
if (strlen($location) && strlen($tester)) {
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
UpdateTester($location, $tester, $testerInfo, $cpu);
}
if (array_key_exists('shard_test', $testInfo) && $testInfo['shard_test']) {
ProcessIncrementalResult();
}
if (array_key_exists('file', $_FILES) && array_key_exists('tmp_name', $_FILES['file'])) {
if (isset($har) && $har) {
ProcessUploadedHAR($testPath);
} else {
logMsg(" Extracting uploaded file '{$_FILES['file']['tmp_name']}' to '{$testPath}'\n");
$archive = new PclZip($_FILES['file']['tmp_name']);
$list = $archive->extract(PCLZIP_OPT_PATH, "{$testPath}/", PCLZIP_OPT_REMOVE_ALL_PATH);
}
}
// compress the text data files
if (isset($_FILES['file'])) {
/**
* Start any instances that may be needed to handle large batches or
* to keep the minimum instance count for a given location
*
*/
function EC2_StartNeededInstances()
{
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!$instances || !is_array($instances)) {
$instances = array();
}
$locations = EC2_GetAMILocations();
$scaleFactor = GetSetting('EC2.ScaleFactor');
if (!$scaleFactor) {
$scaleFactor = 100;
}
// see how long the work queues are for each location in each AMI
foreach ($locations as $ami => $info) {
$tests = 0;
$min = 0;
$max = 1;
foreach ($info['locations'] as $location) {
$queues = GetQueueLengths($location);
if (isset($queues) && is_array($queues)) {
foreach ($queues as $priority => $count) {
$tests += $count;
}
}
$locMin = GetSetting("EC2.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMin = GetSetting("EC2.{$location}.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMax = GetSetting("EC2.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
$locMax = GetSetting("EC2.{$location}.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
}
$locations[$ami]['tests'] = $tests;
$locations[$ami]['min'] = $min;
$locations[$ami]['max'] = $max;
}
foreach ($locations as $ami => $info) {
$count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0;
$target = $locations[$ami]['tests'] / $scaleFactor;
$target = min($target, $locations[$ami]['max']);
$target = max($target, $locations[$ami]['min']);
// See if we have any offline testers that we need to bring online
$online_target = intval($locations[$ami]['tests'] / ($scaleFactor / 2));
foreach ($info['locations'] as $location) {
$testers = GetTesters($location);
if (isset($testers) && is_array($testers) && isset($testers['testers']) && count($testers['testers'])) {
$online = 0;
foreach ($testers['testers'] as $tester) {
if (!isset($tester['offline']) || !$tester['offline']) {
$online++;
}
}
if ($online < $online_target) {
$changed = false;
foreach ($testers['testers'] as $tester) {
if ($online < $online_target && isset($tester['offline']) && $tester['offline']) {
$tester['offline'] = false;
$online++;
UpdateTester($location, $tester['id'], $tester);
}
}
}
}
}
// Start new instances as needed
if ($count < $target) {
$needed = $target - $count;
for ($i = 0; $i < $needed; $i++) {
if (EC2_StartInstance($ami)) {
if (!isset($instances[$ami])) {
$instances[$ami] = array('count' => 0);
}
if (!isset($instances[$ami]['count'])) {
$instances[$ami]['count'] = 0;
}
$instances[$ami]['count']++;
} else {
break;
}
}
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
Unlock($lock);
}
}
$locations = explode(',', $location);
foreach ($locations as $loc) {
$testers = GetTesters($loc);
if (isset($testers) && is_array($testers) && isset($testers['testers'])) {
foreach ($testers['testers'] as &$tester) {
if (array_key_exists('ec2', $tester) && strlen($tester['ec2']) && !$tester['offline']) {
if ($count > 0 && !strlen($tester['test'])) {
$terminate[] = $tester['ec2'];
$count--;
$counts["{$region}.{$ami}"]--;
}
// see if this tester is on the terminate list (for testers that support multiple locations)
foreach ($terminate as $id) {
if ($tester['ec2'] == $id) {
$tester['offline'] = true;
UpdateTester($loc, $tester['id'], $tester);
}
}
}
}
}
}
}
// final step, terminate the instances we don't need
if (!$addOnly) {
$termCount = count($terminate);
echo "Terminating {$termCount} instances running in {$region}...";
if ($termCount) {
$response = $ec2->terminate_instances($terminate);
if ($response->isOK()) {
echo "ok\n";