/** * Load the location information and extract just the end nodes * */ function GetAllTesters() { $locations = array(); $loc = LoadLocationsIni(); if (isset($_REQUEST['location'])) { $location = $_REQUEST['location']; $new = array('locations' => array('1' => 'group', 'default' => 'group'), 'group' => array('1' => $location, 'default' => $location, 'label' => 'placeholder')); if (isset($loc[$_REQUEST['location']])) { $new[$_REQUEST['location']] = $loc[$_REQUEST['location']]; } $loc = $new; } BuildLocations($loc); $i = 1; while (isset($loc['locations'][$i])) { $group =& $loc[$loc['locations'][$i]]; $j = 1; while (isset($group[$j])) { if (array_key_exists('relayServer', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayServer']) && array_key_exists('relayLocation', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayLocation'])) { $locations[$loc[$group[$j]]['location']] = GetRemoteTesters($loc[$group[$j]]['relayServer'], $loc[$group[$j]]['relayLocation']); } else { $locations[$loc[$group[$j]]['location']] = GetTesters($loc[$group[$j]]['location']); } $j++; } $i++; } return $locations; }
/** * See if we need to reboot this tester * */ function GetReboot() { global $location; global $pc; global $ec2; global $tester; $reboot = false; $name = @strlen($ec2) ? $ec2 : $pc; if (isset($name) && strlen($name) && isset($location) && strlen($location)) { $rebootFile = "./work/jobs/{$location}/{$name}.reboot"; if (is_file($rebootFile)) { unlink($rebootFile); $reboot = true; } } // If we have a 100% error rate for the current PC, send it a reboot if (!$reboot) { $testers = GetTesters($location); foreach ($testers as $t) { if ($t['id'] == $tester && !$rebooted) { if ($t['errors'] >= 100) { UpdateTester($location, $tester, null, null, null, true); $reboot = true; } break; } } } if ($reboot) { header('Content-type: text/plain'); header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); echo "Reboot"; } return $reboot; }
/** * Get the backlog for the given directory * * @param mixed $dir */ function GetBacklog($dir, $locationId) { $backlog = array(); $userCount = 0; $lowCount = 0; $testing = 0; $idle = 0; for ($i = 1; $i <= 9; $i++) { $backlog["p{$i}"] = 0; } $queue = GetQueueLengths($locationId); if (count($queue)) { $userCount = $queue[0]; for ($i = 1; $i <= 9; $i++) { $backlog["p{$i}"] = $queue[$i]; $lowCount += $queue[$i]; } } $testers = GetTesters($locationId); if (isset($testers) && is_array($testers) && array_key_exists('testers', $testers)) { foreach ($testers['testers'] as &$tester) { if ($tester['busy']) { $testing++; } else { $idle++; } } } $backlog['Total'] = $userCount + $lowCount + $testing; $backlog['HighPriority'] = $userCount; $backlog['LowPriority'] = $lowCount; $backlog['Testing'] = $testing; $backlog['Idle'] = $idle; return $backlog; }
function GetInstallLocationInfo(&$locations, $location) { $info = array('state' => 'pass', 'label' => "{$location} : ", 'locations' => array()); if (array_key_exists($location, $locations)) { if (array_key_exists('label', $locations[$location])) { $info['label'] .= $locations[$location]['label']; } else { $info['label'] .= '<span class="fail">Label Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } foreach ($locations[$location] as $id => $loc_name) { if (is_numeric($id)) { $info['locations'][$loc_name] = array('state' => 'pass', 'label' => "{$loc_name} : "); if (array_key_exists($loc_name, $locations)) { if (array_key_exists('label', $locations[$loc_name])) { $info['locations'][$loc_name]['label'] .= $locations[$loc_name]['label']; } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">Label Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } $info['locations'][$loc_name]['label'] .= ' - '; $testerCount = 0; $elapsedCheck = -1; $testers = GetTesters($loc_name); if (isset($testers['elapsed'])) { $elapsedCheck = $testers['elapsed']; } if (isset($testers) && is_array($testers) && isset($testers['testers'])) { $testerCount = count($testers['testers']); } if ($testerCount && $elapsedCheck >= 0) { if ($elapsedCheck < 60) { $info['locations'][$loc_name]['label'] .= "<span class=\"pass\">{$testerCount} agents connected</span>"; } else { $info['locations'][$loc_name]['label'] .= "<span class=\"fail\">{$elapsedCheck} minutes since last agent connected</span>"; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">No Agents Connected</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">Definition Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } } } else { $info['label'] .= '<span class="fail">Definition Missing</span>'; $info['state'] = 'fail'; } return $info; }
chdir('..'); include 'common.inc'; error_reporting(E_ALL); // check and see if all of the locations have checked in within the last 30 minutes header('Content-type: text/plain'); header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); $locations = LoadLocationsIni(); $settings = parse_ini_file('./settings/settings.ini', true); $count = 0; $collected = ''; $files = scandir('./tmp'); foreach ($files as $file) { if (is_dir("./tmp/{$file}") && preg_match('/testers-(.+)/', $file, $matches)) { $loc = $matches[1]; $testers = GetTesters($loc, false, false); if (isset($testers['elapsed'])) { $minutes = $testers['elapsed']; if ($minutes < 4320 && isset($locations[$loc]) && !isset($locations[$loc]['hidden'])) { $alert = null; if ($minutes > 60) { $alert = "has not checked for new jobs in {$minutes} minutes."; $collected .= "{$loc} - {$minutes} minutes"; } elseif (isset($locations[$loc]['agents'])) { $configured = $locations[$loc]['agents']; $expected = isset($locations[$loc]['min-agents']) ? $locations[$loc]['min-agents'] : $configured; $tester_count = isset($testers['testers']) ? count($testers['testers']) : 0; if ($tester_count < $expected) { $missing = $configured - $tester_count; $alert = "has {$missing} agents offline ({$tester_count} connected, minimum of {$expected} of the {$configured} required)."; $collected .= "{$loc} - {$missing} agents offline";
/** * Get an actual task to complete * */ function GetJob() { $is_done = false; global $location; global $key; global $pc; global $ec2; global $tester; global $recover; global $is_json; global $dnsServers; $workDir = "./work/jobs/{$location}"; $locKey = GetLocationKey($location); if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) { if ($lock = LockLocation($location)) { $now = time(); $testers = GetTesters($location); // make sure the tester isn't marked as offline (usually when shutting down EC2 instances) $testerCount = isset($testers['testers']) ? count($testers['testers']) : 0; $testerIndex = null; if ($testerCount) { foreach ($testers['testers'] as $index => $testerInfo) { if ($testerInfo['id'] == $tester) { $testerIndex = $index; break; } } } $fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount); if (isset($fileName) && strlen($fileName)) { $is_done = true; $delete = true; if ($is_json) { header("Content-type: application/json"); } else { header('Content-type: text/plain'); } header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); // send the test info to the test agent $testInfo = file_get_contents("{$workDir}/{$fileName}"); // extract the test ID from the job file if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) { $testId = trim($matches[1]); } if (isset($testId)) { // figure out the path to the results $testPath = './' . GetTestPath($testId); // flag the test with the start time $ini = file_get_contents("{$testPath}/testinfo.ini"); if (stripos($ini, 'startTime=') === false) { $time = time(); $start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time); $out = str_replace('[test]', $start, $ini); file_put_contents("{$testPath}/testinfo.ini", $out); } $lock = LockTest($testId); if ($lock) { $testInfoJson = GetTestInfo($testId); if ($testInfoJson) { if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) { $testInfoJson['tester'] = $tester; } if (isset($dnsServers) && strlen($dnsServers)) { $testInfoJson['testerDNS'] = $dnsServers; } if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) { $testInfoJson['started'] = $time; logTestMsg($testId, "Starting test (initiated by tester {$tester})"); } if (!array_key_exists('test_runs', $testInfoJson)) { $testInfoJson['test_runs'] = array(); } for ($run = 1; $run <= $testInfoJson['runs']; $run++) { if (!array_key_exists($run, $testInfoJson['test_runs'])) { $testInfoJson['test_runs'][$run] = array('done' => false); } } $testInfoJson['id'] = $testId; ProcessTestShard($testInfoJson, $testInfo, $delete); SaveTestInfo($testId, $testInfoJson); } UnlockTest($lock); } file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId); } if ($delete) { unlink("{$workDir}/{$fileName}"); } else { AddJobFileHead($workDir, $fileName, $priority, true); } if ($is_json) { $testJson = array(); $script = ''; $isScript = false; $lines = explode("\r\n", $testInfo); foreach ($lines as $line) { if (strlen(trim($line))) { if ($isScript) { if (strlen($script)) { $script .= "\r\n"; } $script .= $line; } elseif (!strcasecmp($line, '[Script]')) { $isScript = true; } else { $pos = strpos($line, '='); if ($pos > -1) { $key = trim(substr($line, 0, $pos)); $value = trim(substr($line, $pos + 1)); if (strlen($key) && strlen($value)) { if (is_numeric($value)) { $testJson[$key] = (int) $value; } else { $testJson[$key] = $value; } } } } } } if (strlen($script)) { $testJson['script'] = $script; } echo json_encode($testJson); } else { echo $testInfo; } $ok = true; } // zero out the tracked page loads in case some got lost if (!$is_done && is_file("./tmp/{$location}.tests")) { $tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true); if ($tests) { $tests['tests'] = 0; file_put_contents("./tmp/{$location}.tests", json_encode($tests)); } } UnlockLocation($lock); // keep track of the last time this location reported in $testerInfo = array(); $testerInfo['ip'] = $_SERVER['REMOTE_ADDR']; $testerInfo['pc'] = $pc; $testerInfo['ec2'] = $ec2; $testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver']; $testerInfo['freedisk'] = @$_GET['freedisk']; $testerInfo['ie'] = @$_GET['ie']; $testerInfo['dns'] = $dnsServers; $testerInfo['video'] = @$_GET['video']; $testerInfo['GPU'] = @$_GET['GPU']; $testerInfo['test'] = ''; if (isset($testId)) { $testerInfo['test'] = $testId; } UpdateTester($location, $tester, $testerInfo); } } return $is_done; }
/** * See if we need to reboot this tester * */ function GetReboot() { global $location; global $pc; global $ec2; global $tester; $reboot = false; $name = @strlen($ec2) ? $ec2 : $pc; if (isset($name) && strlen($name) && isset($location) && strlen($location)) { $rebootFile = "./work/jobs/{$location}/{$name}.reboot"; if (is_file($rebootFile)) { unlink($rebootFile); $reboot = true; } } // If we have a 100% error rate for the current PC, send it a reboot if (!$reboot) { $testers = GetTesters($location); foreach ($testers as $t) { if ($t['id'] == $tester && !$rebooted) { if ($t['errors'] >= 100) { UpdateTester($location, $tester, null, null, null, true); $reboot = true; } break; } } } // If we sent down more than 3 updates sequentially, reboot the tester if (!$reboot && function_exists('apc_fetch') && function_exists('apc_store')) { $updateCount = apc_fetch("uc-{$location}-{$tester}"); if ($updateCount && $updateCount >= 3) { $reboot = true; $updateCount = 0; apc_store("uc-{$location}-{$tester}", $updateCount, 3600); } } if ($reboot) { header('Content-type: text/plain'); header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); echo "Reboot"; } return $reboot; }
/** * For any locations that haven't connected in at least 2 hours, go through and delete any tests in the work queue * */ function CheckLocations() { $locations = LoadLocationsIni(); BuildLocations($locations); $deleted = false; echo "\n"; for ($i = 1; array_key_exists($i, $locations['locations']); $i++) { $group =& $locations[$locations['locations'][$i]]; for ($j = 1; array_key_exists($j, $group); $j++) { if (!array_key_exists('relayServer', $loc[$group[$j]])) { $name = $locations[$group[$j]]['location']; $location = GetTesters($name); $workdir = $locations[$name]['localDir']; $elapsed = -1; if (isset($location) && array_key_exists('elapsed', $location)) { $elapsed = $location['elapsed']; } if ($elapsed < 0 || $elapsed > 120) { if (strlen($workdir)) { if (is_dir($workdir)) { echo "{$elapsed} minutes : {$name} - {$workdir}\n"; delTree($workdir); rmdir($workdir); $deleted = true; } } } } } } // nuke all of the queue files if we had to delete something if ($deleted) { $files = scandir('./tmp'); foreach ($files as $file) { if (stripos($file, '.queue') !== false) { unlink("./tmp/{$file}"); } } } }
/** * Load the location information and extract just the end nodes * */ function GetAllTesters() { $locations = array(); $loc = LoadLocationsIni(); BuildLocations($loc); $i = 1; while (isset($loc['locations'][$i])) { $group =& $loc[$loc['locations'][$i]]; $j = 1; while (isset($group[$j])) { if (array_key_exists('relayServer', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayServer']) && array_key_exists('relayLocation', $loc[$group[$j]]) && strlen($loc[$group[$j]]['relayLocation'])) { $locations[$loc[$group[$j]]['location']] = GetRemoteTesters($loc[$group[$j]]['relayServer'], $loc[$group[$j]]['relayLocation']); } else { $locations[$loc[$group[$j]]['location']] = GetTesters($loc[$group[$j]]['location']); } $j++; } $i++; } return $locations; }
function EC2_GetTesters() { $locations = array(); $loc = LoadLocationsIni(); $i = 1; while (isset($loc['locations'][$i])) { $group =& $loc[$loc['locations'][$i]]; $j = 1; while (isset($group[$j])) { $locations[$group[$j]] = GetTesters($group[$j]); $j++; } $i++; } return $locations; }
$size = $regionData['size']; } $response = $ec2->request_spot_instances($regionData['price'], array('InstanceCount' => (int) $needed, 'Type' => 'one-time', 'LaunchSpecification' => array('ImageId' => $ami, 'InstanceType' => $size, 'UserData' => base64_encode($regionData['userdata'])))); if ($response->isOK()) { echo "ok\n"; $counts["{$region}.{$ami}"] += $needed; } else { echo "failed\n"; } } } elseif ($needed < 0 && !$addOnly) { // lock the location while we mark some free instances for decomm $count = abs($needed); $locations = explode(',', $location); foreach ($locations as $loc) { $testers = GetTesters($loc); if (isset($testers) && is_array($testers) && isset($testers['testers'])) { foreach ($testers['testers'] as &$tester) { if (array_key_exists('ec2', $tester) && strlen($tester['ec2']) && !$tester['offline']) { if ($count > 0 && !strlen($tester['test'])) { $terminate[] = $tester['ec2']; $count--; $counts["{$region}.{$ami}"]--; } // see if this tester is on the terminate list (for testers that support multiple locations) foreach ($terminate as $id) { if ($tester['ec2'] == $id) { $tester['offline'] = true; UpdateTester($loc, $tester['id'], $tester); } }