/**
* Start any instances that may be needed to handle large batches or
* to keep the minimum instance count for a given location
*
*/
function EC2_StartNeededInstances()
{
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!$instances || !is_array($instances)) {
$instances = array();
}
$locations = EC2_GetAMILocations();
$scaleFactor = GetSetting('EC2.ScaleFactor');
if (!$scaleFactor) {
$scaleFactor = 100;
}
// see how long the work queues are for each location in each AMI
foreach ($locations as $ami => $info) {
$tests = 0;
$min = 0;
$max = 1;
foreach ($info['locations'] as $location) {
$queues = GetQueueLengths($location);
if (isset($queues) && is_array($queues)) {
foreach ($queues as $priority => $count) {
$tests += $count;
}
}
$locMin = GetSetting("EC2.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMin = GetSetting("EC2.{$location}.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMax = GetSetting("EC2.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
$locMax = GetSetting("EC2.{$location}.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
}
$locations[$ami]['tests'] = $tests;
$locations[$ami]['min'] = $min;
$locations[$ami]['max'] = $max;
}
foreach ($locations as $ami => $info) {
$count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0;
$target = $locations[$ami]['tests'] / $scaleFactor;
$target = min($target, $locations[$ami]['max']);
$target = max($target, $locations[$ami]['min']);
// See if we have any offline testers that we need to bring online
$online_target = intval($locations[$ami]['tests'] / ($scaleFactor / 2));
foreach ($info['locations'] as $location) {
$lock = LockLocation($location);
if ($lock) {
if (is_file("./tmp/{$location}.tm")) {
$testers = json_decode(file_get_contents("./tmp/{$location}.tm"), true);
if (isset($testers) && is_array($testers) && count($testers)) {
$online = 0;
foreach ($testers as $tester) {
if (!isset($tester['offline']) || !$tester['offline']) {
$online++;
}
}
if ($online < $online_target) {
$changed = false;
foreach ($testers as &$tester) {
if ($online < $online_target && isset($tester['offline']) && $tester['offline']) {
$tester['offline'] = false;
$online++;
$changed = true;
}
}
if ($changed) {
file_put_contents("./tmp/{$location}.tm", json_encode($testers));
}
}
}
}
UnlockLocation($lock);
}
}
// Start new instances as needed
if ($count < $target) {
$needed = $target - $count;
for ($i = 0; $i < $needed; $i++) {
if (EC2_StartInstance($ami)) {
if (!isset($instances[$ami])) {
$instances[$ami] = array('count' => 0);
}
if (!isset($instances[$ami]['count'])) {
$instances[$ami]['count'] = 0;
}
$instances[$ami]['count']++;
} else {
break;
}
}
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
Unlock($lock);
}
}
/**
* Get the backlog for the given directory
*
* @param mixed $dir
*/
function GetBacklog($dir, $locationId)
{
$backlog = array();
$userCount = 0;
$lowCount = 0;
$testing = 0;
$idle = 0;
for ($i = 1; $i <= 9; $i++) {
$backlog["p{$i}"] = 0;
}
$queue = GetQueueLengths($locationId);
if (count($queue)) {
$userCount = $queue[0];
for ($i = 1; $i <= 9; $i++) {
$backlog["p{$i}"] = $queue[$i];
$lowCount += $queue[$i];
}
}
$testers = GetTesters($locationId);
if (isset($testers) && is_array($testers) && array_key_exists('testers', $testers)) {
foreach ($testers['testers'] as &$tester) {
if ($tester['busy']) {
$testing++;
} else {
$idle++;
}
}
}
$backlog['Total'] = $userCount + $lowCount + $testing;
$backlog['HighPriority'] = $userCount;
$backlog['LowPriority'] = $lowCount;
$backlog['Testing'] = $testing;
$backlog['Idle'] = $idle;
return $backlog;
}
/**
* Start any instances that may be needed to handle large batches or
* to keep the minimum instance count for a given location
*
*/
function EC2_StartNeededInstances()
{
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!$instances || !is_array($instances)) {
$instances = array();
}
$locations = EC2_GetAMILocations();
$scaleFactor = GetSetting('EC2.ScaleFactor');
if (!$scaleFactor) {
$scaleFactor = 100;
}
// see how long the work queues are for each location in each AMI
foreach ($locations as $ami => $info) {
$tests = 0;
$min = 0;
$max = 1;
foreach ($info['locations'] as $location) {
$queues = GetQueueLengths($location);
if (isset($queues) && is_array($queues)) {
foreach ($queues as $priority => $count) {
$tests += $count;
}
}
$locMin = GetSetting("EC2.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMin = GetSetting("EC2.{$location}.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMax = GetSetting("EC2.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
$locMax = GetSetting("EC2.{$location}.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
}
$locations[$ami]['tests'] = $tests;
$locations[$ami]['min'] = $min;
$locations[$ami]['max'] = $max;
}
foreach ($locations as $ami => $info) {
$count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0;
$target = $locations[$ami]['tests'] / $scaleFactor;
$target = min($target, $locations[$ami]['max']);
$target = max($target, $locations[$ami]['min']);
$needed = 0;
if ($count < $target) {
$needed = $target - $count;
for ($i = 0; $i < $needed; $i++) {
if (EC2_StartInstance($ami)) {
if (!isset($instances[$ami])) {
$instances[$ami] = array('count' => 0);
}
if (!isset($instances[$ami]['count'])) {
$instances[$ami]['count'] = 0;
}
$instances[$ami]['count']++;
} else {
break;
}
}
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
Unlock($lock);
}
}
