/**
* Load the location information and extract just the end nodes
*
*/
function GetAllTesters()
{
$locations = array();
$loc = LoadLocationsIni();
if (isset($_REQUEST['location'])) {
$location = $_REQUEST['location'];
$new = array('locations' => array('1' => 'group', 'default' => 'group'), 'group' => array('1' => $location, 'default' => $location, 'label' => 'placeholder'));
if (isset($loc[$_REQUEST['location']])) {
$new[$_REQUEST['location']] = $loc[$_REQUEST['location']];
}
$loc = $new;
}
BuildLocations($loc);
$i = 1;
while (isset($loc['locations'][$i])) {
$group =& $loc[$loc['locations'][$i]];
$j = 1;
while (isset($group[$j])) {
if (array_key_exists('relayServer', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayServer']) && array_key_exists('relayLocation', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayLocation'])) {
$locations[$loc[$group[$j]]['location']] = GetRemoteTesters($loc[$group[$j]]['relayServer'], $loc[$group[$j]]['relayLocation']);
} else {
$locations[$loc[$group[$j]]['location']] = GetTesters($loc[$group[$j]]['location']);
}
$j++;
}
$i++;
}
return $locations;
}
/**
* See if we need to reboot this tester
*
*/
function GetReboot()
{
global $location;
global $pc;
global $ec2;
global $tester;
$reboot = false;
$name = @strlen($ec2) ? $ec2 : $pc;
if (isset($name) && strlen($name) && isset($location) && strlen($location)) {
$rebootFile = "./work/jobs/{$location}/{$name}.reboot";
if (is_file($rebootFile)) {
unlink($rebootFile);
$reboot = true;
}
}
// If we have a 100% error rate for the current PC, send it a reboot
if (!$reboot) {
$testers = GetTesters($location);
foreach ($testers as $t) {
if ($t['id'] == $tester && !$rebooted) {
if ($t['errors'] >= 100) {
UpdateTester($location, $tester, null, null, null, true);
$reboot = true;
}
break;
}
}
}
if ($reboot) {
header('Content-type: text/plain');
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
echo "Reboot";
}
return $reboot;
}
/**
* Get the backlog for the given directory
*
* @param mixed $dir
*/
function GetBacklog($dir, $locationId)
{
$backlog = array();
$userCount = 0;
$lowCount = 0;
$testing = 0;
$idle = 0;
for ($i = 1; $i <= 9; $i++) {
$backlog["p{$i}"] = 0;
}
$queue = GetQueueLengths($locationId);
if (count($queue)) {
$userCount = $queue[0];
for ($i = 1; $i <= 9; $i++) {
$backlog["p{$i}"] = $queue[$i];
$lowCount += $queue[$i];
}
}
$testers = GetTesters($locationId);
if (isset($testers) && is_array($testers) && array_key_exists('testers', $testers)) {
foreach ($testers['testers'] as &$tester) {
if ($tester['busy']) {
$testing++;
} else {
$idle++;
}
}
}
$backlog['Total'] = $userCount + $lowCount + $testing;
$backlog['HighPriority'] = $userCount;
$backlog['LowPriority'] = $lowCount;
$backlog['Testing'] = $testing;
$backlog['Idle'] = $idle;
return $backlog;
}
function GetInstallLocationInfo(&$locations, $location)
{
$info = array('state' => 'pass', 'label' => "{$location} : ", 'locations' => array());
if (array_key_exists($location, $locations)) {
if (array_key_exists('label', $locations[$location])) {
$info['label'] .= $locations[$location]['label'];
} else {
$info['label'] .= '<span class="fail">Label Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
foreach ($locations[$location] as $id => $loc_name) {
if (is_numeric($id)) {
$info['locations'][$loc_name] = array('state' => 'pass', 'label' => "{$loc_name} : ");
if (array_key_exists($loc_name, $locations)) {
if (array_key_exists('label', $locations[$loc_name])) {
$info['locations'][$loc_name]['label'] .= $locations[$loc_name]['label'];
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">Label Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
$info['locations'][$loc_name]['label'] .= ' - ';
$testerCount = 0;
$elapsedCheck = -1;
$testers = GetTesters($loc_name);
if (isset($testers['elapsed'])) {
$elapsedCheck = $testers['elapsed'];
}
if (isset($testers) && is_array($testers) && isset($testers['testers'])) {
$testerCount = count($testers['testers']);
}
if ($testerCount && $elapsedCheck >= 0) {
if ($elapsedCheck < 60) {
$info['locations'][$loc_name]['label'] .= "<span class=\"pass\">{$testerCount} agents connected</span>";
} else {
$info['locations'][$loc_name]['label'] .= "<span class=\"fail\">{$elapsedCheck} minutes since last agent connected</span>";
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">No Agents Connected</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">Definition Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
}
}
} else {
$info['label'] .= '<span class="fail">Definition Missing</span>';
$info['state'] = 'fail';
}
return $info;
}
chdir('..');
include 'common.inc';
error_reporting(E_ALL);
// check and see if all of the locations have checked in within the last 30 minutes
header('Content-type: text/plain');
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
$locations = LoadLocationsIni();
$settings = parse_ini_file('./settings/settings.ini', true);
$count = 0;
$collected = '';
$files = scandir('./tmp');
foreach ($files as $file) {
if (is_dir("./tmp/{$file}") && preg_match('/testers-(.+)/', $file, $matches)) {
$loc = $matches[1];
$testers = GetTesters($loc, false, false);
if (isset($testers['elapsed'])) {
$minutes = $testers['elapsed'];
if ($minutes < 4320 && isset($locations[$loc]) && !isset($locations[$loc]['hidden'])) {
$alert = null;
if ($minutes > 60) {
$alert = "has not checked for new jobs in {$minutes} minutes.";
$collected .= "{$loc} - {$minutes} minutes";
} elseif (isset($locations[$loc]['agents'])) {
$configured = $locations[$loc]['agents'];
$expected = isset($locations[$loc]['min-agents']) ? $locations[$loc]['min-agents'] : $configured;
$tester_count = isset($testers['testers']) ? count($testers['testers']) : 0;
if ($tester_count < $expected) {
$missing = $configured - $tester_count;
$alert = "has {$missing} agents offline ({$tester_count} connected, minimum of {$expected} of the {$configured} required).";
$collected .= "{$loc} - {$missing} agents offline";
/**
* Get an actual task to complete
*
*/
function GetJob()
{
$is_done = false;
global $location;
global $key;
global $pc;
global $ec2;
global $tester;
global $recover;
global $is_json;
global $dnsServers;
$workDir = "./work/jobs/{$location}";
$locKey = GetLocationKey($location);
if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) {
if ($lock = LockLocation($location)) {
$now = time();
$testers = GetTesters($location);
// make sure the tester isn't marked as offline (usually when shutting down EC2 instances)
$testerCount = isset($testers['testers']) ? count($testers['testers']) : 0;
$testerIndex = null;
if ($testerCount) {
foreach ($testers['testers'] as $index => $testerInfo) {
if ($testerInfo['id'] == $tester) {
$testerIndex = $index;
break;
}
}
}
$fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount);
if (isset($fileName) && strlen($fileName)) {
$is_done = true;
$delete = true;
if ($is_json) {
header("Content-type: application/json");
} else {
header('Content-type: text/plain');
}
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
// send the test info to the test agent
$testInfo = file_get_contents("{$workDir}/{$fileName}");
// extract the test ID from the job file
if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) {
$testId = trim($matches[1]);
}
if (isset($testId)) {
// figure out the path to the results
$testPath = './' . GetTestPath($testId);
// flag the test with the start time
$ini = file_get_contents("{$testPath}/testinfo.ini");
if (stripos($ini, 'startTime=') === false) {
$time = time();
$start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time);
$out = str_replace('[test]', $start, $ini);
file_put_contents("{$testPath}/testinfo.ini", $out);
}
$lock = LockTest($testId);
if ($lock) {
$testInfoJson = GetTestInfo($testId);
if ($testInfoJson) {
if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) {
$testInfoJson['tester'] = $tester;
}
if (isset($dnsServers) && strlen($dnsServers)) {
$testInfoJson['testerDNS'] = $dnsServers;
}
if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) {
$testInfoJson['started'] = $time;
logTestMsg($testId, "Starting test (initiated by tester {$tester})");
}
if (!array_key_exists('test_runs', $testInfoJson)) {
$testInfoJson['test_runs'] = array();
}
for ($run = 1; $run <= $testInfoJson['runs']; $run++) {
if (!array_key_exists($run, $testInfoJson['test_runs'])) {
$testInfoJson['test_runs'][$run] = array('done' => false);
}
}
$testInfoJson['id'] = $testId;
ProcessTestShard($testInfoJson, $testInfo, $delete);
SaveTestInfo($testId, $testInfoJson);
}
UnlockTest($lock);
}
file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId);
}
if ($delete) {
unlink("{$workDir}/{$fileName}");
} else {
AddJobFileHead($workDir, $fileName, $priority, true);
}
if ($is_json) {
$testJson = array();
$script = '';
$isScript = false;
$lines = explode("\r\n", $testInfo);
foreach ($lines as $line) {
if (strlen(trim($line))) {
if ($isScript) {
if (strlen($script)) {
$script .= "\r\n";
}
$script .= $line;
} elseif (!strcasecmp($line, '[Script]')) {
$isScript = true;
} else {
$pos = strpos($line, '=');
if ($pos > -1) {
$key = trim(substr($line, 0, $pos));
$value = trim(substr($line, $pos + 1));
if (strlen($key) && strlen($value)) {
if (is_numeric($value)) {
$testJson[$key] = (int) $value;
} else {
$testJson[$key] = $value;
}
}
}
}
}
}
if (strlen($script)) {
$testJson['script'] = $script;
}
echo json_encode($testJson);
} else {
echo $testInfo;
}
$ok = true;
}
// zero out the tracked page loads in case some got lost
if (!$is_done && is_file("./tmp/{$location}.tests")) {
$tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true);
if ($tests) {
$tests['tests'] = 0;
file_put_contents("./tmp/{$location}.tests", json_encode($tests));
}
}
UnlockLocation($lock);
// keep track of the last time this location reported in
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
$testerInfo['pc'] = $pc;
$testerInfo['ec2'] = $ec2;
$testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver'];
$testerInfo['freedisk'] = @$_GET['freedisk'];
$testerInfo['ie'] = @$_GET['ie'];
$testerInfo['dns'] = $dnsServers;
$testerInfo['video'] = @$_GET['video'];
$testerInfo['GPU'] = @$_GET['GPU'];
$testerInfo['test'] = '';
if (isset($testId)) {
$testerInfo['test'] = $testId;
}
UpdateTester($location, $tester, $testerInfo);
}
}
return $is_done;
}
/**
* See if we need to reboot this tester
*
*/
function GetReboot()
{
global $location;
global $pc;
global $ec2;
global $tester;
$reboot = false;
$name = @strlen($ec2) ? $ec2 : $pc;
if (isset($name) && strlen($name) && isset($location) && strlen($location)) {
$rebootFile = "./work/jobs/{$location}/{$name}.reboot";
if (is_file($rebootFile)) {
unlink($rebootFile);
$reboot = true;
}
}
// If we have a 100% error rate for the current PC, send it a reboot
if (!$reboot) {
$testers = GetTesters($location);
foreach ($testers as $t) {
if ($t['id'] == $tester && !$rebooted) {
if ($t['errors'] >= 100) {
UpdateTester($location, $tester, null, null, null, true);
$reboot = true;
}
break;
}
}
}
// If we sent down more than 3 updates sequentially, reboot the tester
if (!$reboot && function_exists('apc_fetch') && function_exists('apc_store')) {
$updateCount = apc_fetch("uc-{$location}-{$tester}");
if ($updateCount && $updateCount >= 3) {
$reboot = true;
$updateCount = 0;
apc_store("uc-{$location}-{$tester}", $updateCount, 3600);
}
}
if ($reboot) {
header('Content-type: text/plain');
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
echo "Reboot";
}
return $reboot;
}
/**
* For any locations that haven't connected in at least 2 hours, go through and delete any tests in the work queue
*
*/
function CheckLocations()
{
$locations = LoadLocationsIni();
BuildLocations($locations);
$deleted = false;
echo "\n";
for ($i = 1; array_key_exists($i, $locations['locations']); $i++) {
$group =& $locations[$locations['locations'][$i]];
for ($j = 1; array_key_exists($j, $group); $j++) {
if (!array_key_exists('relayServer', $loc[$group[$j]])) {
$name = $locations[$group[$j]]['location'];
$location = GetTesters($name);
$workdir = $locations[$name]['localDir'];
$elapsed = -1;
if (isset($location) && array_key_exists('elapsed', $location)) {
$elapsed = $location['elapsed'];
}
if ($elapsed < 0 || $elapsed > 120) {
if (strlen($workdir)) {
if (is_dir($workdir)) {
echo "{$elapsed} minutes : {$name} - {$workdir}\n";
delTree($workdir);
rmdir($workdir);
$deleted = true;
}
}
}
}
}
}
// nuke all of the queue files if we had to delete something
if ($deleted) {
$files = scandir('./tmp');
foreach ($files as $file) {
if (stripos($file, '.queue') !== false) {
unlink("./tmp/{$file}");
}
}
}
}
/**
* Load the location information and extract just the end nodes
*
*/
function GetAllTesters()
{
$locations = array();
$loc = LoadLocationsIni();
BuildLocations($loc);
$i = 1;
while (isset($loc['locations'][$i])) {
$group =& $loc[$loc['locations'][$i]];
$j = 1;
while (isset($group[$j])) {
if (array_key_exists('relayServer', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayServer']) && array_key_exists('relayLocation', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayLocation'])) {
$locations[$loc[$group[$j]]['location']] = GetRemoteTesters($loc[$group[$j]]['relayServer'], $loc[$group[$j]]['relayLocation']);
} else {
$locations[$loc[$group[$j]]['location']] = GetTesters($loc[$group[$j]]['location']);
}
$j++;
}
$i++;
}
return $locations;
}
function EC2_GetTesters()
{
$locations = array();
$loc = LoadLocationsIni();
$i = 1;
while (isset($loc['locations'][$i])) {
$group =& $loc[$loc['locations'][$i]];
$j = 1;
while (isset($group[$j])) {
$locations[$group[$j]] = GetTesters($group[$j]);
$j++;
}
$i++;
}
return $locations;
}
$size = $regionData['size'];
}
$response = $ec2->request_spot_instances($regionData['price'], array('InstanceCount' => (int) $needed, 'Type' => 'one-time', 'LaunchSpecification' => array('ImageId' => $ami, 'InstanceType' => $size, 'UserData' => base64_encode($regionData['userdata']))));
if ($response->isOK()) {
echo "ok\n";
$counts["{$region}.{$ami}"] += $needed;
} else {
echo "failed\n";
}
}
} elseif ($needed < 0 && !$addOnly) {
// lock the location while we mark some free instances for decomm
$count = abs($needed);
$locations = explode(',', $location);
foreach ($locations as $loc) {
$testers = GetTesters($loc);
if (isset($testers) && is_array($testers) && isset($testers['testers'])) {
foreach ($testers['testers'] as &$tester) {
if (array_key_exists('ec2', $tester) && strlen($tester['ec2']) && !$tester['offline']) {
if ($count > 0 && !strlen($tester['test'])) {
$terminate[] = $tester['ec2'];
$count--;
$counts["{$region}.{$ami}"]--;
}
// see if this tester is on the terminate list (for testers that support multiple locations)
foreach ($terminate as $id) {
if ($tester['ec2'] == $id) {
$tester['offline'] = true;
UpdateTester($loc, $tester['id'], $tester);
}
}
