function ResubmitTest($id)
{
echo "{$id} - Resubmitting...";
$testPath = './' . GetTestPath($id);
if (gz_is_file("{$testPath}/testinfo.json")) {
$test = json_decode(gz_file_get_contents("{$testPath}/testinfo.json"), true);
if (array_key_exists('job_file', $test) && array_key_exists('location', $test) && is_file("{$testPath}/test.job")) {
if ($lock = LockLocation($test['location'])) {
if (copy("{$testPath}/test.job", $test['job_file'])) {
$files = scandir($testPath);
foreach ($files as $file) {
if ($file != '.' && $file != '..' && strncasecmp($file, 'test', 4)) {
if (is_file("{$testPath}/{$file}")) {
unlink("{$testPath}/{$file}");
} elseif (is_dir("{$testPath}/{$file}")) {
delTree("{$testPath}/{$file}");
}
}
}
AddJobFile($test['workdir'], $test['job'], $test['priority'], false);
$test['started'] = time();
unset($test['completeTime']);
gz_file_put_contents("{$testPath}/testinfo.json", json_encode($test));
echo "OK";
} else {
echo "Failed to copy job file";
}
UnlockLocation($lock);
} else {
echo "Failed to lock location";
}
} else {
echo "Invalid test";
}
} else {
echo "Test not found";
}
echo "\n";
}
$size = $regionData['size'];
}
$response = $ec2->request_spot_instances($regionData['price'], array('InstanceCount' => (int) $needed, 'Type' => 'one-time', 'LaunchSpecification' => array('ImageId' => $ami, 'InstanceType' => $size, 'UserData' => base64_encode($regionData['userdata']))));
if ($response->isOK()) {
echo "ok\n";
$counts["{$region}.{$ami}"] += $needed;
} else {
echo "failed\n";
}
}
} elseif ($needed < 0 && !$addOnly) {
// lock the location while we mark some free instances for decomm
$count = abs($needed);
$locations = explode(',', $location);
foreach ($locations as $loc) {
if ($lock = LockLocation($loc)) {
$testers = json_decode(file_get_contents("./tmp/{$loc}.tm"), true);
if (count($testers)) {
foreach ($testers as &$tester) {
if (array_key_exists('ec2', $tester) && strlen($tester['ec2']) && !$tester['offline']) {
if ($count > 0 && !strlen($tester['test'])) {
$terminate[] = $tester['ec2'];
$count--;
$counts["{$region}.{$ami}"]--;
}
// see if this tester is on the terminate list (for testers that support multiple locations)
foreach ($terminate as $id) {
if ($tester['ec2'] == $id) {
$tester['offline'] = true;
}
}
/**
* Write out the actual job file
*/
function WriteJob($location, &$test, &$job, $testId)
{
$ret = false;
global $error;
global $locations;
if ($locations[$location]['relayServer']) {
// upload the test to a the relay server
$test['id'] = $testId;
$ret = SendToRelay($test, $job);
} else {
// make sure the work directory exists
if (!is_dir($test['workdir'])) {
mkdir($test['workdir'], 0777, true);
}
$workDir = $test['workdir'];
$locationLock = LockLocation($location);
if (isset($locationLock)) {
if (isset($test['affinity'])) {
$test['job'] = "Affinity{$test['affinity']}.{$test['job']}";
}
$fileName = $test['job'];
$file = "{$workDir}/{$fileName}";
if (file_put_contents($file, $job)) {
if (AddJobFile($workDir, $fileName, $test['priority'], $test['queue_limit'])) {
// store a copy of the job file with the original test in case the test fails and we need to resubmit it
$test['job_file'] = realpath($file);
if (ValidateTestId($testId)) {
$testPath = GetTestPath($testId);
if (strlen($testPath)) {
$testPath = './' . $testPath;
if (!is_dir($testPath)) {
mkdir($testPath, 0777, true);
}
file_put_contents("{$testPath}/test.job", $job);
}
}
$tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true);
if (!$tests) {
$tests = array();
}
$testCount = $test['runs'];
if (!$test['fvonly']) {
$testCount *= 2;
}
if (array_key_exists('tests', $tests)) {
$tests['tests'] += $testCount;
} else {
$tests['tests'] = $testCount;
}
file_put_contents("./tmp/{$location}.tests", json_encode($tests));
$ret = true;
} else {
unlink($file);
$error = "Sorry, that test location already has too many tests pending. Pleasy try again later.";
}
}
UnlockLocation($locationLock);
}
}
return $ret;
}
/**
* Handle sharded test results where they come in individually
*
*/
function ProcessIncrementalResult()
{
global $testPath;
global $done;
global $testInfo;
global $testInfo_dirty;
global $runNumber;
global $cacheWarmed;
global $location;
if ($done) {
// mark this test as done
$testInfo['test_runs'][$runNumber]['done'] = true;
$testInfo_dirty = true;
// make sure all of the sharded tests are done
for ($run = 1; $run <= $testInfo['runs'] && $done; $run++) {
if (!$testInfo['test_runs'][$run]['done']) {
$done = false;
}
}
if (!$done && array_key_exists('discarded', $testInfo['test_runs'][$runNumber]) && $testInfo['test_runs'][$runNumber]['discarded']) {
if (is_file("{$testPath}/test.job")) {
if ($lock = LockLocation($location)) {
if (copy("{$testPath}/test.job", $testInfo['job_file'])) {
AddJobFileHead($testInfo['workdir'], $testInfo['job'], $testInfo['priority'], true);
}
UnlockLocation($lock);
}
}
}
}
}
/**
* Get an actual task to complete
*
*/
function GetJob()
{
$is_done = false;
global $location;
global $key;
global $pc;
global $ec2;
global $tester;
global $recover;
global $is_json;
global $dnsServers;
$workDir = "./work/jobs/{$location}";
$locKey = GetLocationKey($location);
if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) {
if ($lock = LockLocation($location)) {
$now = time();
$testers = GetTesters($location, true);
// make sure the tester isn't marked as offline (usually when shutting down EC2 instances)
$testerCount = isset($testers['testers']) ? count($testers['testers']) : 0;
$testerIndex = null;
$offline = false;
if ($testerCount) {
if (strlen($ec2)) {
foreach ($testers['testers'] as $index => $testerInfo) {
if (isset($testerInfo['ec2']) && $testerInfo['ec2'] == $ec2 && isset($testerInfo['offline']) && $testerInfo['offline']) {
$offline = true;
}
break;
}
}
foreach ($testers['testers'] as $index => $testerInfo) {
if ($testerInfo['id'] == $tester) {
$testerIndex = $index;
break;
}
}
}
if (!$offline) {
$fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount);
if (isset($fileName) && strlen($fileName)) {
$is_done = true;
$delete = true;
if ($is_json) {
header("Content-type: application/json");
} else {
header('Content-type: text/plain');
}
header("Cache-Control: no-cache, must-revalidate");
header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
// send the test info to the test agent
$testInfo = file_get_contents("{$workDir}/{$fileName}");
// extract the test ID from the job file
if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) {
$testId = trim($matches[1]);
}
if (isset($testId)) {
// figure out the path to the results
$testPath = './' . GetTestPath($testId);
// flag the test with the start time
$ini = file_get_contents("{$testPath}/testinfo.ini");
if (stripos($ini, 'startTime=') === false) {
$time = time();
$start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time);
$out = str_replace('[test]', $start, $ini);
file_put_contents("{$testPath}/testinfo.ini", $out);
}
$lock = LockTest($testId);
if ($lock) {
$testInfoJson = GetTestInfo($testId);
if ($testInfoJson) {
if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) {
$testInfoJson['tester'] = $tester;
}
if (isset($dnsServers) && strlen($dnsServers)) {
$testInfoJson['testerDNS'] = $dnsServers;
}
if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) {
$testInfoJson['started'] = $time;
logTestMsg($testId, "Starting test (initiated by tester {$tester})");
}
if (!array_key_exists('test_runs', $testInfoJson)) {
$testInfoJson['test_runs'] = array();
}
for ($run = 1; $run <= $testInfoJson['runs']; $run++) {
if (!array_key_exists($run, $testInfoJson['test_runs'])) {
$testInfoJson['test_runs'][$run] = array('done' => false);
}
}
$testInfoJson['id'] = $testId;
ProcessTestShard($testInfoJson, $testInfo, $delete);
SaveTestInfo($testId, $testInfoJson);
}
UnlockTest($lock);
}
file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId);
}
if ($delete) {
unlink("{$workDir}/{$fileName}");
} else {
AddJobFileHead($workDir, $fileName, $priority, true);
}
if ($is_json) {
$testJson = array();
$script = '';
$isScript = false;
$lines = explode("\r\n", $testInfo);
foreach ($lines as $line) {
if (strlen(trim($line))) {
if ($isScript) {
if (strlen($script)) {
$script .= "\r\n";
}
$script .= $line;
} elseif (!strcasecmp($line, '[Script]')) {
$isScript = true;
} else {
$pos = strpos($line, '=');
if ($pos !== false) {
$key = trim(substr($line, 0, $pos));
$value = trim(substr($line, $pos + 1));
if (strlen($key) && strlen($value)) {
if ($key == 'customMetric') {
$pos = strpos($value, ':');
if ($pos !== false) {
$metric = trim(substr($value, 0, $pos));
$code = base64_decode(substr($value, $pos + 1));
if ($code !== false && strlen($metric) && strlen($code)) {
if (!isset($testJson['customMetrics'])) {
$testJson['customMetrics'] = array();
}
$testJson['customMetrics'][$metric] = $code;
}
}
} elseif (is_numeric($value)) {
$testJson[$key] = (int) $value;
} else {
$testJson[$key] = $value;
}
}
}
}
}
}
if (strlen($script)) {
$testJson['script'] = $script;
}
echo json_encode($testJson);
} else {
echo $testInfo;
}
$ok = true;
}
// zero out the tracked page loads in case some got lost
if (!$is_done && is_file("./tmp/{$location}.tests")) {
$tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true);
if ($tests) {
$tests['tests'] = 0;
file_put_contents("./tmp/{$location}.tests", json_encode($tests));
}
}
}
UnlockLocation($lock);
// keep track of the last time this location reported in
$testerInfo = array();
$testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
$testerInfo['pc'] = $pc;
$testerInfo['ec2'] = $ec2;
$testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver'];
$testerInfo['freedisk'] = @$_GET['freedisk'];
$testerInfo['ie'] = @$_GET['ie'];
$testerInfo['dns'] = $dnsServers;
$testerInfo['video'] = @$_GET['video'];
$testerInfo['GPU'] = @$_GET['GPU'];
$testerInfo['test'] = '';
if (isset($testId)) {
$testerInfo['test'] = $testId;
}
UpdateTester($location, $tester, $testerInfo);
}
}
return $is_done;
}
function GetLocationInfo(&$locations, $location)
{
$info = array('state' => 'pass', 'label' => "{$location} : ", 'video' => false, 'locations' => array());
if (array_key_exists($location, $locations)) {
if (array_key_exists('label', $locations[$location])) {
$info['label'] .= $locations[$location]['label'];
} else {
$info['label'] .= '<span class="fail">Label Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
foreach ($locations[$location] as $id => $loc_name) {
if (is_numeric($id)) {
$info['locations'][$loc_name] = array('state' => 'pass', 'label' => "{$loc_name} : ");
if (array_key_exists($loc_name, $locations)) {
if (array_key_exists('label', $locations[$loc_name])) {
$info['locations'][$loc_name]['label'] .= $locations[$loc_name]['label'];
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">Label Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
$info['locations'][$loc_name]['label'] .= ' - ';
$file = "./tmp/{$loc_name}.tm";
$testerCount = 0;
$elapsedCheck = -1;
$lock = LockLocation($loc_name);
if ($lock) {
if (is_file($file)) {
$now = time();
$testers = json_decode(file_get_contents($file), true);
$testerCount = count($testers);
if ($testerCount) {
$latest = 0;
foreach ($testers as $tester) {
if (array_key_exists('updated', $tester) && $tester['updated'] > $latest) {
$latest = $tester['updated'];
}
if (array_key_exists('video', $tester) && $tester['video']) {
$info['video'] = true;
}
}
if ($latest > 0) {
$elapsedCheck = 0;
if ($now > $latest) {
$elapsedCheck = (int) (($now - $latest) / 60);
}
}
}
}
UnlockLocation($lock);
}
if ($testerCount && $elapsedCheck >= 0) {
if ($elapsedCheck < 60) {
$info['locations'][$loc_name]['label'] .= "<span class=\"pass\">{$testerCount} agents connected</span>";
} else {
$info['locations'][$loc_name]['label'] .= "<span class=\"fail\">{$elapsedCheck} minutes since last agent connected</span>";
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">No Agents Connected</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
} else {
$info['locations'][$loc_name]['label'] .= '<span class="fail">Definition Missing</span>';
$info['locations'][$loc_name]['state'] = 'fail';
$info['state'] = 'fail';
}
}
}
} else {
$info['label'] .= '<span class="fail">Definition Missing</span>';
$info['state'] = 'fail';
}
return $info;
}
/**
* Start any instances that may be needed to handle large batches or
* to keep the minimum instance count for a given location
*
*/
function EC2_StartNeededInstances()
{
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!$instances || !is_array($instances)) {
$instances = array();
}
$locations = EC2_GetAMILocations();
$scaleFactor = GetSetting('EC2.ScaleFactor');
if (!$scaleFactor) {
$scaleFactor = 100;
}
// see how long the work queues are for each location in each AMI
foreach ($locations as $ami => $info) {
$tests = 0;
$min = 0;
$max = 1;
foreach ($info['locations'] as $location) {
$queues = GetQueueLengths($location);
if (isset($queues) && is_array($queues)) {
foreach ($queues as $priority => $count) {
$tests += $count;
}
}
$locMin = GetSetting("EC2.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMin = GetSetting("EC2.{$location}.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMax = GetSetting("EC2.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
$locMax = GetSetting("EC2.{$location}.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
}
$locations[$ami]['tests'] = $tests;
$locations[$ami]['min'] = $min;
$locations[$ami]['max'] = $max;
}
foreach ($locations as $ami => $info) {
$count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0;
$target = $locations[$ami]['tests'] / $scaleFactor;
$target = min($target, $locations[$ami]['max']);
$target = max($target, $locations[$ami]['min']);
// See if we have any offline testers that we need to bring online
$online_target = intval($locations[$ami]['tests'] / ($scaleFactor / 2));
foreach ($info['locations'] as $location) {
$lock = LockLocation($location);
if ($lock) {
if (is_file("./tmp/{$location}.tm")) {
$testers = json_decode(file_get_contents("./tmp/{$location}.tm"), true);
if (isset($testers) && is_array($testers) && count($testers)) {
$online = 0;
foreach ($testers as $tester) {
if (!isset($tester['offline']) || !$tester['offline']) {
$online++;
}
}
if ($online < $online_target) {
$changed = false;
foreach ($testers as &$tester) {
if ($online < $online_target && isset($tester['offline']) && $tester['offline']) {
$tester['offline'] = false;
$online++;
$changed = true;
}
}
if ($changed) {
file_put_contents("./tmp/{$location}.tm", json_encode($testers));
}
}
}
}
UnlockLocation($lock);
}
}
// Start new instances as needed
if ($count < $target) {
$needed = $target - $count;
for ($i = 0; $i < $needed; $i++) {
if (EC2_StartInstance($ami)) {
if (!isset($instances[$ami])) {
$instances[$ami] = array('count' => 0);
}
if (!isset($instances[$ami]['count'])) {
$instances[$ami]['count'] = 0;
}
$instances[$ami]['count']++;
} else {
break;
}
}
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
Unlock($lock);
}
}
