public function OnCheckRecovered(CheckRecoveredEvent $event)
{
$serverAlerts = new Alerts($event->dBServer);
$hasActiveAlert = $serverAlerts->hasActiveAlert($event->check);
if ($hasActiveAlert) {
$serverAlerts->solveAlert($event->check);
}
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
try {
$dbFarm = DBFarm::LoadByID($farmId);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
return;
}
} catch (Exception $e) {
return;
}
foreach ($dbFarm->GetFarmRoles() as $dbFarmRole) {
$instancesHealth = array();
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($dbFarm->EnvID);
$aws = $env->aws($dbFarmRole);
$statusList = $aws->ec2->instance->describeStatus();
foreach ($statusList as $sd) {
/* @var $sd \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$instancesHealth[$sd->instanceId] = $sd;
}
unset($statusList);
//Releases memory
$env->getContainer()->release('aws');
unset($aws);
}
$servers = $this->db->Execute("\n SELECT server_id FROM servers\n WHERE farm_roleid = ? AND status = ?\n ", array($dbFarmRole->ID, SERVER_STATUS::RUNNING));
while ($server = $servers->FetchRow()) {
$dbServer = DBServer::LoadByID($server['server_id']);
// Do not support ami-scripts
if (!$dbServer->IsSupported("0.5")) {
continue;
}
// Do not support windows
if ($dbServer->IsSupported("0.8") && !$dbServer->IsSupported("0.9")) {
continue;
}
$subStatus = $dbServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS);
if ($subStatus != '') {
if ($subStatus == 'stopped') {
//Need to solve ALL failed metrics
$serverAlerts = new Alerts($dbServer);
if ($serverAlerts->getActiveAlertsCount()) {
$serverAlerts->solveAlert();
}
}
continue;
}
if ($dbServer->GetProperty(SERVER_PROPERTIES::REBOOTING)) {
continue;
}
$serverAlerts = new Alerts($dbServer);
//Check AWS healthchecks
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
try {
/* @var $statusInfo \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$statusInfo = isset($instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)]) ? $instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)] : null;
if ($statusInfo) {
$check = Alerts::METRIC_AWS_SYSTEM_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->systemStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($statusInfo->systemStatus->status != 'ok' && $statusInfo->systemStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->systemStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->systemStatus->status}: {$txtDetails}"));
}
$check = Alerts::METRIC_AWS_INSTANCE_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->instanceStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} else {
if ($statusInfo->instanceStatus->status != 'ok' && $statusInfo->instanceStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->instanceStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->instanceStatus->status}: {$txtDetails}"));
}
}
unset($statusInfo);
}
} catch (Exception $e) {
}
}
//Not supported by VPC yet.
if ($dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
continue;
}
//Check scalr-upd-client status
$check = Alerts::METRIC_SCALARIZR_UPD_CLIENT_CONNECTIVITY;
$port = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_UPDC_PORT);
if (!$port) {
$port = 8008;
}
$result = $this->checkPort($dbServer->remoteIp, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
//Check scalarizr connectivity status
$check = Alerts::METRIC_SCALARIZR_CONNECTIVITY;
$port = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT);
if (!$port) {
$port = 8013;
}
$result = $this->checkPort($dbServer->remoteIp, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
}
}
exit;
}
