function ResubmitTest($id) { echo "{$id} - Resubmitting..."; $testPath = './' . GetTestPath($id); if (gz_is_file("{$testPath}/testinfo.json")) { $test = json_decode(gz_file_get_contents("{$testPath}/testinfo.json"), true); if (array_key_exists('job_file', $test) && array_key_exists('location', $test) && is_file("{$testPath}/test.job")) { if ($lock = LockLocation($test['location'])) { if (copy("{$testPath}/test.job", $test['job_file'])) { $files = scandir($testPath); foreach ($files as $file) { if ($file != '.' && $file != '..' && strncasecmp($file, 'test', 4)) { if (is_file("{$testPath}/{$file}")) { unlink("{$testPath}/{$file}"); } elseif (is_dir("{$testPath}/{$file}")) { delTree("{$testPath}/{$file}"); } } } AddJobFile($test['workdir'], $test['job'], $test['priority'], false); $test['started'] = time(); unset($test['completeTime']); gz_file_put_contents("{$testPath}/testinfo.json", json_encode($test)); echo "OK"; } else { echo "Failed to copy job file"; } UnlockLocation($lock); } else { echo "Failed to lock location"; } } else { echo "Invalid test"; } } else { echo "Test not found"; } echo "\n"; }
if ($count > 0 && !strlen($tester['test'])) { $terminate[] = $tester['ec2']; $count--; $counts["{$region}.{$ami}"]--; } // see if this tester is on the terminate list (for testers that support multiple locations) foreach ($terminate as $id) { if ($tester['ec2'] == $id) { $tester['offline'] = true; } } } } file_put_contents("./tmp/{$loc}.tm", json_encode($testers)); } UnlockLocation($lock); } } } // final step, terminate the instances we don't need if (!$addOnly) { $termCount = count($terminate); echo "Terminating {$termCount} instances running in {$region}..."; if ($termCount) { $response = $ec2->terminate_instances($terminate); if ($response->isOK()) { echo "ok\n"; } else { echo "failed\n"; } } else {
/** * Write out the actual job file */ function WriteJob($location, &$test, &$job, $testId) { $ret = false; global $error; global $locations; if ($locations[$location]['relayServer']) { // upload the test to a the relay server $test['id'] = $testId; $ret = SendToRelay($test, $job); } else { // make sure the work directory exists if (!is_dir($test['workdir'])) { mkdir($test['workdir'], 0777, true); } $workDir = $test['workdir']; $locationLock = LockLocation($location); if (isset($locationLock)) { if (isset($test['affinity'])) { $test['job'] = "Affinity{$test['affinity']}.{$test['job']}"; } $fileName = $test['job']; $file = "{$workDir}/{$fileName}"; if (file_put_contents($file, $job)) { if (AddJobFile($workDir, $fileName, $test['priority'], $test['queue_limit'])) { // store a copy of the job file with the original test in case the test fails and we need to resubmit it $test['job_file'] = realpath($file); if (ValidateTestId($testId)) { $testPath = GetTestPath($testId); if (strlen($testPath)) { $testPath = './' . $testPath; if (!is_dir($testPath)) { mkdir($testPath, 0777, true); } file_put_contents("{$testPath}/test.job", $job); } } $tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true); if (!$tests) { $tests = array(); } $testCount = $test['runs']; if (!$test['fvonly']) { $testCount *= 2; } if (array_key_exists('tests', $tests)) { $tests['tests'] += $testCount; } else { $tests['tests'] = $testCount; } file_put_contents("./tmp/{$location}.tests", json_encode($tests)); $ret = true; } else { unlink($file); $error = "Sorry, that test location already has too many tests pending. Pleasy try again later."; } } UnlockLocation($locationLock); } } return $ret; }
/** * Handle sharded test results where they come in individually * */ function ProcessIncrementalResult() { global $testPath; global $done; global $testInfo; global $testInfo_dirty; global $runNumber; global $cacheWarmed; global $location; if ($done) { // mark this test as done $testInfo['test_runs'][$runNumber]['done'] = true; $testInfo_dirty = true; // make sure all of the sharded tests are done for ($run = 1; $run <= $testInfo['runs'] && $done; $run++) { if (!$testInfo['test_runs'][$run]['done']) { $done = false; } } if (!$done && array_key_exists('discarded', $testInfo['test_runs'][$runNumber]) && $testInfo['test_runs'][$runNumber]['discarded']) { if (is_file("{$testPath}/test.job")) { if ($lock = LockLocation($location)) { if (copy("{$testPath}/test.job", $testInfo['job_file'])) { AddJobFileHead($testInfo['workdir'], $testInfo['job'], $testInfo['priority'], true); } UnlockLocation($lock); } } } } }
/** * Get an actual task to complete * */ function GetJob() { $is_done = false; global $location; global $key; global $pc; global $ec2; global $tester; global $recover; global $is_json; global $dnsServers; $workDir = "./work/jobs/{$location}"; $locKey = GetLocationKey($location); if (strpos($location, '..') == false && strpos($location, '\\') == false && strpos($location, '/') == false && (!strlen($locKey) || !strcmp($key, $locKey))) { if ($lock = LockLocation($location)) { $now = time(); $testers = GetTesters($location, true); // make sure the tester isn't marked as offline (usually when shutting down EC2 instances) $testerCount = isset($testers['testers']) ? count($testers['testers']) : 0; $testerIndex = null; $offline = false; if ($testerCount) { if (strlen($ec2)) { foreach ($testers['testers'] as $index => $testerInfo) { if (isset($testerInfo['ec2']) && $testerInfo['ec2'] == $ec2 && isset($testerInfo['offline']) && $testerInfo['offline']) { $offline = true; } break; } } foreach ($testers['testers'] as $index => $testerInfo) { if ($testerInfo['id'] == $tester) { $testerIndex = $index; break; } } } if (!$offline) { $fileName = GetJobFile($workDir, $priority, $pc, $testerIndex, $testerCount); if (isset($fileName) && strlen($fileName)) { $is_done = true; $delete = true; if ($is_json) { header("Content-type: application/json"); } else { header('Content-type: text/plain'); } header("Cache-Control: no-cache, must-revalidate"); header("Expires: Sat, 26 Jul 1997 05:00:00 GMT"); // send the test info to the test agent $testInfo = file_get_contents("{$workDir}/{$fileName}"); // extract the test ID from the job file if (preg_match('/Test ID=([^\\r\\n]+)\\r/i', $testInfo, $matches)) { $testId = trim($matches[1]); } if (isset($testId)) { // figure out the path to the results $testPath = './' . GetTestPath($testId); // flag the test with the start time $ini = file_get_contents("{$testPath}/testinfo.ini"); if (stripos($ini, 'startTime=') === false) { $time = time(); $start = "[test]\r\nstartTime=" . gmdate("m/d/y G:i:s", $time); $out = str_replace('[test]', $start, $ini); file_put_contents("{$testPath}/testinfo.ini", $out); } $lock = LockTest($testId); if ($lock) { $testInfoJson = GetTestInfo($testId); if ($testInfoJson) { if (!array_key_exists('tester', $testInfoJson) || !strlen($testInfoJson['tester'])) { $testInfoJson['tester'] = $tester; } if (isset($dnsServers) && strlen($dnsServers)) { $testInfoJson['testerDNS'] = $dnsServers; } if (!array_key_exists('started', $testInfoJson) || !$testInfoJson['started']) { $testInfoJson['started'] = $time; logTestMsg($testId, "Starting test (initiated by tester {$tester})"); } if (!array_key_exists('test_runs', $testInfoJson)) { $testInfoJson['test_runs'] = array(); } for ($run = 1; $run <= $testInfoJson['runs']; $run++) { if (!array_key_exists($run, $testInfoJson['test_runs'])) { $testInfoJson['test_runs'][$run] = array('done' => false); } } $testInfoJson['id'] = $testId; ProcessTestShard($testInfoJson, $testInfo, $delete); SaveTestInfo($testId, $testInfoJson); } UnlockTest($lock); } file_put_contents("./tmp/last-test-{$location}-{$tester}.test", $testId); } if ($delete) { unlink("{$workDir}/{$fileName}"); } else { AddJobFileHead($workDir, $fileName, $priority, true); } if ($is_json) { $testJson = array(); $script = ''; $isScript = false; $lines = explode("\r\n", $testInfo); foreach ($lines as $line) { if (strlen(trim($line))) { if ($isScript) { if (strlen($script)) { $script .= "\r\n"; } $script .= $line; } elseif (!strcasecmp($line, '[Script]')) { $isScript = true; } else { $pos = strpos($line, '='); if ($pos !== false) { $key = trim(substr($line, 0, $pos)); $value = trim(substr($line, $pos + 1)); if (strlen($key) && strlen($value)) { if ($key == 'customMetric') { $pos = strpos($value, ':'); if ($pos !== false) { $metric = trim(substr($value, 0, $pos)); $code = base64_decode(substr($value, $pos + 1)); if ($code !== false && strlen($metric) && strlen($code)) { if (!isset($testJson['customMetrics'])) { $testJson['customMetrics'] = array(); } $testJson['customMetrics'][$metric] = $code; } } } elseif (is_numeric($value)) { $testJson[$key] = (int) $value; } else { $testJson[$key] = $value; } } } } } } if (strlen($script)) { $testJson['script'] = $script; } echo json_encode($testJson); } else { echo $testInfo; } $ok = true; } // zero out the tracked page loads in case some got lost if (!$is_done && is_file("./tmp/{$location}.tests")) { $tests = json_decode(file_get_contents("./tmp/{$location}.tests"), true); if ($tests) { $tests['tests'] = 0; file_put_contents("./tmp/{$location}.tests", json_encode($tests)); } } } UnlockLocation($lock); // keep track of the last time this location reported in $testerInfo = array(); $testerInfo['ip'] = $_SERVER['REMOTE_ADDR']; $testerInfo['pc'] = $pc; $testerInfo['ec2'] = $ec2; $testerInfo['ver'] = array_key_exists('version', $_GET) ? $_GET['version'] : $_GET['ver']; $testerInfo['freedisk'] = @$_GET['freedisk']; $testerInfo['ie'] = @$_GET['ie']; $testerInfo['dns'] = $dnsServers; $testerInfo['video'] = @$_GET['video']; $testerInfo['GPU'] = @$_GET['GPU']; $testerInfo['test'] = ''; if (isset($testId)) { $testerInfo['test'] = $testId; } UpdateTester($location, $tester, $testerInfo); } } return $is_done; }
function GetLocationInfo(&$locations, $location) { $info = array('state' => 'pass', 'label' => "{$location} : ", 'video' => false, 'locations' => array()); if (array_key_exists($location, $locations)) { if (array_key_exists('label', $locations[$location])) { $info['label'] .= $locations[$location]['label']; } else { $info['label'] .= '<span class="fail">Label Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } foreach ($locations[$location] as $id => $loc_name) { if (is_numeric($id)) { $info['locations'][$loc_name] = array('state' => 'pass', 'label' => "{$loc_name} : "); if (array_key_exists($loc_name, $locations)) { if (array_key_exists('label', $locations[$loc_name])) { $info['locations'][$loc_name]['label'] .= $locations[$loc_name]['label']; } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">Label Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } $info['locations'][$loc_name]['label'] .= ' - '; $file = "./tmp/{$loc_name}.tm"; $testerCount = 0; $elapsedCheck = -1; $lock = LockLocation($loc_name); if ($lock) { if (is_file($file)) { $now = time(); $testers = json_decode(file_get_contents($file), true); $testerCount = count($testers); if ($testerCount) { $latest = 0; foreach ($testers as $tester) { if (array_key_exists('updated', $tester) && $tester['updated'] > $latest) { $latest = $tester['updated']; } if (array_key_exists('video', $tester) && $tester['video']) { $info['video'] = true; } } if ($latest > 0) { $elapsedCheck = 0; if ($now > $latest) { $elapsedCheck = (int) (($now - $latest) / 60); } } } } UnlockLocation($lock); } if ($testerCount && $elapsedCheck >= 0) { if ($elapsedCheck < 60) { $info['locations'][$loc_name]['label'] .= "<span class=\"pass\">{$testerCount} agents connected</span>"; } else { $info['locations'][$loc_name]['label'] .= "<span class=\"fail\">{$elapsedCheck} minutes since last agent connected</span>"; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">No Agents Connected</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } else { $info['locations'][$loc_name]['label'] .= '<span class="fail">Definition Missing</span>'; $info['locations'][$loc_name]['state'] = 'fail'; $info['state'] = 'fail'; } } } } else { $info['label'] .= '<span class="fail">Definition Missing</span>'; $info['state'] = 'fail'; } return $info; }
/** * Start any instances that may be needed to handle large batches or * to keep the minimum instance count for a given location * */ function EC2_StartNeededInstances() { $lock = Lock('ec2-instances', true, 120); if ($lock) { $instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true); if (!$instances || !is_array($instances)) { $instances = array(); } $locations = EC2_GetAMILocations(); $scaleFactor = GetSetting('EC2.ScaleFactor'); if (!$scaleFactor) { $scaleFactor = 100; } // see how long the work queues are for each location in each AMI foreach ($locations as $ami => $info) { $tests = 0; $min = 0; $max = 1; foreach ($info['locations'] as $location) { $queues = GetQueueLengths($location); if (isset($queues) && is_array($queues)) { foreach ($queues as $priority => $count) { $tests += $count; } } $locMin = GetSetting("EC2.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMin = GetSetting("EC2.{$location}.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMax = GetSetting("EC2.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } $locMax = GetSetting("EC2.{$location}.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } } $locations[$ami]['tests'] = $tests; $locations[$ami]['min'] = $min; $locations[$ami]['max'] = $max; } foreach ($locations as $ami => $info) { $count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0; $target = $locations[$ami]['tests'] / $scaleFactor; $target = min($target, $locations[$ami]['max']); $target = max($target, $locations[$ami]['min']); // See if we have any offline testers that we need to bring online $online_target = intval($locations[$ami]['tests'] / ($scaleFactor / 2)); foreach ($info['locations'] as $location) { $lock = LockLocation($location); if ($lock) { if (is_file("./tmp/{$location}.tm")) { $testers = json_decode(file_get_contents("./tmp/{$location}.tm"), true); if (isset($testers) && is_array($testers) && count($testers)) { $online = 0; foreach ($testers as $tester) { if (!isset($tester['offline']) || !$tester['offline']) { $online++; } } if ($online < $online_target) { $changed = false; foreach ($testers as &$tester) { if ($online < $online_target && isset($tester['offline']) && $tester['offline']) { $tester['offline'] = false; $online++; $changed = true; } } if ($changed) { file_put_contents("./tmp/{$location}.tm", json_encode($testers)); } } } } UnlockLocation($lock); } } // Start new instances as needed if ($count < $target) { $needed = $target - $count; for ($i = 0; $i < $needed; $i++) { if (EC2_StartInstance($ami)) { if (!isset($instances[$ami])) { $instances[$ami] = array('count' => 0); } if (!isset($instances[$ami]['count'])) { $instances[$ami]['count'] = 0; } $instances[$ami]['count']++; } else { break; } } } } file_put_contents('./tmp/ec2-instances.dat', json_encode($instances)); Unlock($lock); } }