function EC2_GetRunningInstances()
{
$now = time();
$instances = array();
$key = GetSetting('ec2_key');
$secret = GetSetting('ec2_secret');
if ($key && $secret) {
$locations = EC2_GetAMILocations();
try {
$ec2 = \Aws\Ec2\Ec2Client::factory(array('key' => $key, 'secret' => $secret, 'region' => 'us-east-1'));
$regions = array();
$response = $ec2->describeRegions();
if (isset($response['Regions'])) {
foreach ($response['Regions'] as $region) {
$regions[] = $region['RegionName'];
}
}
} catch (\Aws\Ec2\Exception\Ec2Exception $e) {
$error = $e->getMessage();
EC2LogError("Listing running EC2 instances: {$error}");
}
if (isset($regions) && is_array($regions) && count($regions)) {
foreach ($regions as $region) {
try {
$ec2 = \Aws\Ec2\Ec2Client::factory(array('key' => $key, 'secret' => $secret, 'region' => $region));
$response = $ec2->describeInstances();
if (isset($response['Reservations'])) {
foreach ($response['Reservations'] as $reservation) {
foreach ($reservation['Instances'] as $instance) {
$wptLocations = null;
// See what locations are associated with the AMI
if (isset($instance['ImageId']) && isset($locations[$instance['ImageId']]['locations'])) {
$wptLocations = $locations[$instance['ImageId']]['locations'];
} elseif (isset($instance['Tags'])) {
// fall back to using tags to identify locations if they were set
foreach ($instance['Tags'] as $tag) {
if ($tag['Key'] == 'WPTLocations') {
$wptLocations = explode(',', $tag['Value']);
break;
}
}
}
if (isset($wptLocations)) {
$launchTime = strtotime($instance['LaunchTime']);
$elapsed = $now - $launchTime;
$state = $instance['State']['Code'];
$running = false;
if (is_numeric($state) && $state <= 16) {
$running = true;
}
$instances[] = array('region' => $region, 'id' => $instance['InstanceId'], 'ami' => $instance['ImageId'], 'state' => $state, 'launchTime' => $instance['LaunchTime'], 'launched' => $launchTime, 'runningTime' => $elapsed, 'locations' => $wptLocations, 'running' => $running);
}
}
}
}
} catch (\Aws\Ec2\Exception\Ec2Exception $e) {
$error = $e->getMessage();
EC2LogError("Listing running EC2 instances: {$error}");
}
}
}
}
// update the AMI counts we are tracking locally
if (count($instances)) {
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$amis = array();
foreach ($instances as $instance) {
if (isset($instance['ami']) && strlen($instance['ami']) && $instance['running']) {
if (!isset($amis[$instance['ami']])) {
$amis[$instance['ami']] = array('count' => 0);
}
$amis[$instance['ami']]['count']++;
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($amis));
Unlock($lock);
}
}
return $instances;
}
/**
* Start any instances that may be needed to handle large batches or
* to keep the minimum instance count for a given location
*
*/
function EC2_StartNeededInstances()
{
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$instances = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!$instances || !is_array($instances)) {
$instances = array();
}
$locations = EC2_GetAMILocations();
$scaleFactor = GetSetting('EC2.ScaleFactor');
if (!$scaleFactor) {
$scaleFactor = 100;
}
// see how long the work queues are for each location in each AMI
foreach ($locations as $ami => $info) {
$tests = 0;
$min = 0;
$max = 1;
foreach ($info['locations'] as $location) {
$queues = GetQueueLengths($location);
if (isset($queues) && is_array($queues)) {
foreach ($queues as $priority => $count) {
$tests += $count;
}
}
$locMin = GetSetting("EC2.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMin = GetSetting("EC2.{$location}.min");
if ($locMin !== false) {
$min = max(0, intval($locMin));
}
$locMax = GetSetting("EC2.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
$locMax = GetSetting("EC2.{$location}.max");
if ($locMax !== false) {
$max = max(1, intval($locMax));
}
}
$locations[$ami]['tests'] = $tests;
$locations[$ami]['min'] = $min;
$locations[$ami]['max'] = $max;
}
foreach ($locations as $ami => $info) {
$count = isset($instances[$ami]['count']) ? $instances[$ami]['count'] : 0;
$target = $locations[$ami]['tests'] / $scaleFactor;
$target = min($target, $locations[$ami]['max']);
$target = max($target, $locations[$ami]['min']);
$needed = 0;
if ($count < $target) {
$needed = $target - $count;
for ($i = 0; $i < $needed; $i++) {
if (EC2_StartInstance($ami)) {
if (!isset($instances[$ami])) {
$instances[$ami] = array('count' => 0);
}
if (!isset($instances[$ami]['count'])) {
$instances[$ami]['count'] = 0;
}
$instances[$ami]['count']++;
} else {
break;
}
}
}
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
Unlock($lock);
}
}
