/**
* Terminate any EC2 Instances that are configured for auto-scaling
* if they have not had work in the last 15 minutes and are close
* to an hourly increment of running (since EC2 bills hourly)
*
*/
function EC2_TerminateIdleInstances()
{
$instances = EC2_GetRunningInstances();
if (count($instances)) {
$locations = EC2_GetTesters();
foreach ($instances as $instance) {
$minutes = $instance['runningTime'] / 60.0;
if ($minutes > 15 && $minutes % 60 >= 50) {
$terminate = true;
$lastWork = null;
// last job assigned from this location
$lastCheck = null;
// time since this instance connected (if ever)
foreach ($instance['locations'] as $location) {
if (isset($locations[$location]['testers'])) {
foreach ($locations[$location]['testers'] as $tester) {
if (isset($tester['last']) && (!isset($lastWork) || $tester['last'] < $lastWork)) {
$lastWork = $tester['last'];
}
if (isset($tester['ec2']) && $tester['ec2'] == $instance['id']) {
$lastCheck = $tester['elapsed'];
}
}
}
}
if (isset($lastWork) && isset($lastCheck)) {
// Keep the instance if the location had work in the last 15 minutes
// and if this instance has checked in recently
if ($lastWork < 15 && $lastCheck < 15) {
$terminate = false;
}
}
if ($terminate) {
if (isset($instance['ami'])) {
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$counts = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if ($counts && is_array($counts) && isset($counts[$instance['ami']])) {
unset($counts[$instance['ami']]);
file_put_contents('./tmp/ec2-instances.dat', json_encode($instances));
}
Unlock($lock);
}
}
EC2_TerminateInstance($instance['region'], $instance['id']);
}
}
}
}
}
/**
* Terminate any EC2 Instances that are configured for auto-scaling
* if they have not had work in the last 15 minutes and are close
* to an hourly increment of running (since EC2 bills hourly)
*
*/
function EC2_TerminateIdleInstances()
{
EC2_SendInstancesOffline();
$instances = EC2_GetRunningInstances();
if (count($instances)) {
$instanceCounts = array();
$agentCounts = array();
$locations = EC2_GetTesters();
// Do a first pass to count the number of instances at each location/ami
foreach ($instances as $instance) {
if (isset($instance['ami'])) {
if (!isset($instanceCounts[$instance['ami']])) {
$instanceCounts[$instance['ami']] = array('count' => 0);
}
if ($instance['running']) {
$instanceCounts[$instance['ami']]['count']++;
}
}
foreach ($instance['locations'] as $location) {
if (!isset($agentCounts[$location])) {
$agentCounts[$location] = array('min' => 0, 'count' => 0);
$min = GetSetting("EC2.min");
if ($min) {
$agentCounts[$location]['min'] = $min;
}
$min = GetSetting("EC2.{$location}.min");
if ($min) {
$agentCounts[$location]['min'] = $min;
}
}
$agentCounts[$location]['count']++;
}
}
foreach ($instances as $instance) {
$minutes = $instance['runningTime'] / 60.0;
if ($minutes > 15 && $minutes % 60 >= 50) {
$terminate = true;
$lastWork = null;
// last job assigned from this location
$lastCheck = null;
// time since this instance connected (if ever)
foreach ($instance['locations'] as $location) {
if ($agentCounts[$location]['count'] <= $agentCounts[$location]['min']) {
$terminate = false;
} elseif (isset($locations[$location]['testers'])) {
foreach ($locations[$location]['testers'] as $tester) {
if (isset($tester['ec2']) && $tester['ec2'] == $instance['id']) {
if (isset($tester['last']) && (!isset($lastWork) || $tester['last'] < $lastWork)) {
$lastWork = $tester['last'];
}
$lastCheck = $tester['elapsed'];
}
}
}
}
// Keep the instance if the location had work in the last 15 minutes
// and if this instance has checked in recently
if (isset($lastWork) && isset($lastCheck) && $lastWork < 15 && $lastCheck < 15) {
$terminate = false;
}
if ($terminate) {
if (isset($instance['ami']) && $instance['running']) {
$instanceCounts[$instance['ami']]['count']--;
}
EC2_TerminateInstance($instance['region'], $instance['id']);
}
}
}
// update the running instance counts
$lock = Lock('ec2-instances', true, 120);
if ($lock) {
$counts = json_decode(file_get_contents('./tmp/ec2-instances.dat'), true);
if (!isset($counts) || !is_array($counts)) {
$counts = array();
}
foreach ($counts as $ami => $count) {
if (!isset($counts[$ami])) {
$counts[$ami] = array('count' => 0);
}
$counts[$ami]['count'] = isset($instanceCounts[$ami]['count']) ? $instanceCounts[$ami]['count'] : 0;
}
foreach ($instanceCounts as $ami => $count) {
if (!isset($counts[$ami])) {
$counts[$ami] = array('count' => 0);
}
$counts[$ami]['count'] = $count['count'];
}
file_put_contents('./tmp/ec2-instances.dat', json_encode($counts));
Unlock($lock);
}
}
}
