/** * Start any instances that may be needed to handle large batches or * to keep the minimum instance count for a given location * */ function EC2_StartNeededInstances() { $lock = Lock('ec2-instances', true, 120); if ($lock) { $instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true); if (!$instances || !is_array($instances)) { $instances = array(); } $locations = EC2_GetAMILocations(); $scaleFactor = GetSetting('EC2.ScaleFactor'); if (!$scaleFactor) { $scaleFactor = 100; } // see how long the work queues are for each location in each AMI foreach ($locations as $ami => $info) { $tests = 0; $min = 0; $max = 1; foreach ($info['locations'] as $location) { $queues = GetQueueLengths($location); if (isset($queues) && is_array($queues)) { foreach ($queues as $priority => $count) { $tests += $count; } } $locMin = GetSetting("EC2.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMin = GetSetting("EC2.{$location}.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMax = GetSetting("EC2.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } $locMax = GetSetting("EC2.{$location}.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } } $locations[$ami]['tests'] = $tests; $locations[$ami]['min'] = $min; $locations[$ami]['max'] = $max; } foreach ($locations as $ami => $info) { $count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0; $target = $locations[$ami]['tests'] / $scaleFactor; $target = min($target, $locations[$ami]['max']); $target = max($target, $locations[$ami]['min']); // See if we have any offline testers that we need to bring online $online_target = intval($locations[$ami]['tests'] / ($scaleFactor / 2)); foreach ($info['locations'] as $location) { $lock = LockLocation($location); if ($lock) { if (is_file("./tmp/{$location}.tm")) { $testers = json_decode(file_get_contents("./tmp/{$location}.tm"), true); if (isset($testers) && is_array($testers) && count($testers)) { $online = 0; foreach ($testers as $tester) { if (!isset($tester['offline']) || !$tester['offline']) { $online++; } } if ($online < $online_target) { $changed = false; foreach ($testers as &$tester) { if ($online < $online_target && isset($tester['offline']) && $tester['offline']) { $tester['offline'] = false; $online++; $changed = true; } } if ($changed) { file_put_contents("./tmp/{$location}.tm", json_encode($testers)); } } } } UnlockLocation($lock); } } // Start new instances as needed if ($count < $target) { $needed = $target - $count; for ($i = 0; $i < $needed; $i++) { if (EC2_StartInstance($ami)) { if (!isset($instances[$ami])) { $instances[$ami] = array('count' => 0); } if (!isset($instances[$ami]['count'])) { $instances[$ami]['count'] = 0; } $instances[$ami]['count']++; } else { break; } } } } file_put_contents('./tmp/ec2-instances.dat', json_encode($instances)); Unlock($lock); } }
/** * Get the backlog for the given directory * * @param mixed $dir */ function GetBacklog($dir, $locationId) { $backlog = array(); $userCount = 0; $lowCount = 0; $testing = 0; $idle = 0; for ($i = 1; $i <= 9; $i++) { $backlog["p{$i}"] = 0; } $queue = GetQueueLengths($locationId); if (count($queue)) { $userCount = $queue[0]; for ($i = 1; $i <= 9; $i++) { $backlog["p{$i}"] = $queue[$i]; $lowCount += $queue[$i]; } } $testers = GetTesters($locationId); if (isset($testers) && is_array($testers) && array_key_exists('testers', $testers)) { foreach ($testers['testers'] as &$tester) { if ($tester['busy']) { $testing++; } else { $idle++; } } } $backlog['Total'] = $userCount + $lowCount + $testing; $backlog['HighPriority'] = $userCount; $backlog['LowPriority'] = $lowCount; $backlog['Testing'] = $testing; $backlog['Idle'] = $idle; return $backlog; }
/** * Start any instances that may be needed to handle large batches or * to keep the minimum instance count for a given location * */ function EC2_StartNeededInstances() { $lock = Lock('ec2-instances', true, 120); if ($lock) { $instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true); if (!$instances || !is_array($instances)) { $instances = array(); } $locations = EC2_GetAMILocations(); $scaleFactor = GetSetting('EC2.ScaleFactor'); if (!$scaleFactor) { $scaleFactor = 100; } // see how long the work queues are for each location in each AMI foreach ($locations as $ami => $info) { $tests = 0; $min = 0; $max = 1; foreach ($info['locations'] as $location) { $queues = GetQueueLengths($location); if (isset($queues) && is_array($queues)) { foreach ($queues as $priority => $count) { $tests += $count; } } $locMin = GetSetting("EC2.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMin = GetSetting("EC2.{$location}.min"); if ($locMin !== false) { $min = max(0, intval($locMin)); } $locMax = GetSetting("EC2.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } $locMax = GetSetting("EC2.{$location}.max"); if ($locMax !== false) { $max = max(1, intval($locMax)); } } $locations[$ami]['tests'] = $tests; $locations[$ami]['min'] = $min; $locations[$ami]['max'] = $max; } foreach ($locations as $ami => $info) { $count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0; $target = $locations[$ami]['tests'] / $scaleFactor; $target = min($target, $locations[$ami]['max']); $target = max($target, $locations[$ami]['min']); $needed = 0; if ($count < $target) { $needed = $target - $count; for ($i = 0; $i < $needed; $i++) { if (EC2_StartInstance($ami)) { if (!isset($instances[$ami])) { $instances[$ami] = array('count' => 0); } if (!isset($instances[$ami]['count'])) { $instances[$ami]['count'] = 0; } $instances[$ami]['count']++; } else { break; } } } } file_put_contents('./tmp/ec2-instances.dat', json_encode($instances)); Unlock($lock); } }