public function OnCheckRecovered(CheckRecoveredEvent $event)
{
$serverAlerts = new Alerts($event->dBServer);
$hasActiveAlert = $serverAlerts->hasActiveAlert($event->check);
if ($hasActiveAlert) {
$serverAlerts->solveAlert($event->check);
}
}
public function xListAction($serverId = null, $farmId = null, $farmRoleId = null, $status = null)
{
$sql = "SELECT sa.* FROM server_alerts sa LEFT JOIN farms f ON f.id = sa.farm_id WHERE sa.env_id = ?";
$args = [$this->getEnvironmentId()];
if ($serverId) {
$sql .= ' AND sa.server_id = ?';
$args[] = $serverId;
}
if ($farmId) {
$sql .= ' AND sa.farm_id = ?';
$args[] = $farmId;
if ($farmRoleId) {
$sql .= ' AND sa.farm_roleid = ?';
$args[] = $farmRoleId;
}
}
if ($status) {
$sql .= ' AND sa.status = ?';
$args[] = $status;
}
list($sql, $args) = $this->request->prepareFarmSqlQuery($sql, $args, 'f');
$response = $this->buildResponseFromSql2($sql, ['metric', 'status', 'dtoccured', 'dtlastcheck', 'dtsolved', 'details'], ['server_id', 'details'], $args);
foreach ($response['data'] as $i => $row) {
$row['dtoccured'] = Scalr_Util_DateTime::convertTz($row['dtoccured']);
if ($row['dtlastcheck']) {
$row['dtlastcheck'] = Scalr_Util_DateTime::convertTz($row['dtlastcheck']);
} else {
$row['dtlastcheck'] = false;
}
if ($row['status'] == Alerts::STATUS_RESOLVED) {
$row['dtsolved'] = Scalr_Util_DateTime::convertTz($row['dtsolved']);
} else {
$row['dtsolved'] = false;
}
$row['metric'] = Alerts::getMetricName($row['metric']);
$row['farm_name'] = DBFarm::LoadByID($row['farm_id'])->Name;
try {
$row['role_name'] = DBFarmRole::LoadByID($row['farm_roleid'])->GetRoleObject()->name;
$dbServer = DBServer::LoadByID($row['server_id']);
$row['server_exists'] = $dbServer->status == SERVER_STATUS::RUNNING ? true : false;
} catch (Exception $e) {
}
$response['data'][$i] = $row;
}
$this->response->data($response);
}
public function xListAlertsAction()
{
$this->request->defineParams(array('sort' => array('type' => 'string', 'default' => 'id'), 'dir' => array('type' => 'string', 'default' => 'DESC')));
$sql = "SELECT * FROM server_alerts WHERE env_id='" . $this->getEnvironmentId() . "'";
if ($this->getParam('serverId')) {
$sql .= ' AND server_id = ' . $this->db->qstr($this->getParam('serverId'));
}
if ($this->getParam('farmId')) {
$sql .= ' AND farm_id = ' . $this->db->qstr($this->getParam('farmId'));
if ($this->getParam('farmRoleId')) {
$sql .= ' AND farm_roleid = ' . $this->db->qstr($this->getParam('farmRoleId'));
}
}
if ($this->getParam('status')) {
$sql .= ' AND status = ' . $this->db->qstr($this->getParam('status'));
}
$response = $this->buildResponseFromSql($sql, array('server_id', 'details'));
foreach ($response['data'] as $i => $row) {
$row['dtoccured'] = Scalr_Util_DateTime::convertTz($row['dtoccured']);
if ($row['dtlastcheck']) {
$row['dtlastcheck'] = Scalr_Util_DateTime::convertTz($row['dtlastcheck']);
} else {
$row['dtlastcheck'] = false;
}
if ($row['status'] == Alerts::STATUS_RESOLVED) {
$row['dtsolved'] = Scalr_Util_DateTime::convertTz($row['dtsolved']);
} else {
$row['dtsolved'] = false;
}
$row['metric'] = Alerts::getMetricName($row['metric']);
$row['farm_name'] = DBFarm::LoadByID($row['farm_id'])->Name;
try {
$row['role_name'] = DBFarmRole::LoadByID($row['farm_roleid'])->GetRoleObject()->name;
$dbServer = DBServer::LoadByID($row['server_id']);
$row['server_exists'] = $dbServer->status == SERVER_STATUS::RUNNING ? true : false;
} catch (Exception $e) {
}
$response['data'][$i] = $row;
}
$this->response->data($response);
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
try {
$dbFarm = DBFarm::LoadByID($farmId);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
return;
}
} catch (Exception $e) {
return;
}
foreach ($dbFarm->GetFarmRoles() as $dbFarmRole) {
$instancesHealth = array();
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($dbFarm->EnvID);
$aws = $env->aws($dbFarmRole);
$statusList = $aws->ec2->instance->describeStatus();
foreach ($statusList as $sd) {
/* @var $sd \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$instancesHealth[$sd->instanceId] = $sd;
}
unset($statusList);
//Releases memory
$env->getContainer()->release('aws');
unset($aws);
}
$servers = $this->db->Execute("\n SELECT server_id FROM servers\n WHERE farm_roleid = ? AND status = ?\n ", array($dbFarmRole->ID, SERVER_STATUS::RUNNING));
while ($server = $servers->FetchRow()) {
$dbServer = DBServer::LoadByID($server['server_id']);
// Do not support ami-scripts
if (!$dbServer->IsSupported("0.5")) {
continue;
}
// Do not support windows
if ($dbServer->IsSupported("0.8") && !$dbServer->IsSupported("0.9")) {
continue;
}
if ($dbServer->GetProperty(SERVER_PROPERTIES::REBOOTING)) {
continue;
}
$serverAlerts = new Alerts($dbServer);
//Check AWS healthchecks
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
try {
/* @var $statusInfo \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$statusInfo = isset($instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)]) ? $instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)] : null;
if ($statusInfo) {
$check = Alerts::METRIC_AWS_SYSTEM_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->systemStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($statusInfo->systemStatus->status != 'ok' && $statusInfo->systemStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->systemStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->systemStatus->status}: {$txtDetails}"));
}
$check = Alerts::METRIC_AWS_INSTANCE_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->instanceStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} else {
if ($statusInfo->instanceStatus->status != 'ok' && $statusInfo->instanceStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->instanceStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->instanceStatus->status}: {$txtDetails}"));
}
}
unset($statusInfo);
}
} catch (Exception $e) {
}
}
//Not supported by VPC yet.
if ($dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
continue;
}
$ip = $dbServer->getSzrHost();
//Check scalr-upd-client status
$check = Alerts::METRIC_SCALARIZR_UPD_CLIENT_CONNECTIVITY;
$port = $dbServer->getPort(DBServer::PORT_UPDC);
$result = $this->checkPort($ip, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
//Check scalarizr connectivity status
$check = Alerts::METRIC_SCALARIZR_CONNECTIVITY;
$port = $dbServer->getPort(DBServer::PORT_CTRL);
$result = $this->checkPort($ip, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
}
}
exit;
}
public function xListServersAction()
{
if (!$this->request->isAllowed(Acl::RESOURCE_FARMS_SERVERS) && !$this->request->isAllowed(Acl::RESOURCE_FARMS_ROLES, Acl::PERM_FARMS_ROLES_CREATE)) {
throw new Scalr_Exception_InsufficientPermissions();
}
$this->request->defineParams(array('roleId' => array('type' => 'int'), 'farmId' => array('type' => 'int'), 'farmRoleId' => array('type' => 'int'), 'serverId', 'hideTerminated' => array('type' => 'bool'), 'sort' => array('type' => 'json')));
$sql = 'SELECT servers.*, farms.name AS farm_name, roles.name AS role_name, farm_roles.alias AS role_alias
FROM servers
LEFT JOIN farms ON servers.farm_id = farms.id
LEFT JOIN roles ON roles.id = servers.role_id
LEFT JOIN farm_roles ON farm_roles.id = servers.farm_roleid
WHERE servers.env_id = ? AND :FILTER:';
$args = array($this->getEnvironmentId());
if ($this->getParam('cloudServerId')) {
$sql = str_replace('WHERE', 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id WHERE', $sql);
$sql .= ' AND (';
$sql .= 'server_properties.name = ? AND server_properties.value = ?';
$args[] = CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EC2_SERVER_PROPERTIES::INSTANCE_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EUCA_SERVER_PROPERTIES::INSTANCE_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = GCE_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = OPENSTACK_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = RACKSPACE_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ')';
}
if ($this->getParam('cloudServerLocation')) {
if (!strstr($sql, 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id')) {
$sql = str_replace('WHERE', 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id WHERE', $sql);
}
$sql .= ' AND (';
$sql .= 'server_properties.name = ? AND server_properties.value = ?';
$args[] = CLOUDSTACK_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EC2_SERVER_PROPERTIES::REGION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EUCA_SERVER_PROPERTIES::REGION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = GCE_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = RACKSPACE_SERVER_PROPERTIES::DATACENTER;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ')';
}
if ($this->getParam('farmId')) {
$sql .= " AND farm_id=?";
$args[] = $this->getParam('farmId');
}
if ($this->request->isAllowed(Acl::RESOURCE_FARMS_SERVERS)) {
if (!$this->request->isAllowed(Acl::RESOURCE_FARMS, Acl::PERM_FARMS_NOT_OWNED_FARMS)) {
$sql .= " AND (farms.created_by_id = ? OR servers.status IN (?, ?) AND farms.id IS NULL)";
$args[] = $this->user->getId();
$args[] = SERVER_STATUS::IMPORTING;
$args[] = SERVER_STATUS::TEMPORARY;
}
} else {
//show servers related to role creation process only
$sql .= ' AND servers.status IN (?, ?)';
$args[] = SERVER_STATUS::IMPORTING;
$args[] = SERVER_STATUS::TEMPORARY;
}
if ($this->getParam('farmRoleId')) {
$sql .= " AND farm_roleid=?";
$args[] = $this->getParam('farmRoleId');
}
if ($this->getParam('roleId')) {
$sql .= " AND role_id=?";
$args[] = $this->getParam('roleId');
}
if ($this->getParam('serverId')) {
$sql .= " AND server_id=?";
$args[] = $this->getParam('serverId');
}
if ($this->getParam('hideTerminated')) {
$sql .= ' AND servers.status != ?';
$args[] = SERVER_STATUS::TERMINATED;
}
$response = $this->buildResponseFromSql2($sql, array('platform', 'farm_name', 'role_name', 'role_alias', 'index', 'server_id', 'remote_ip', 'local_ip', 'uptime', 'status'), array('servers.server_id', 'farm_id', 'farms.name', 'remote_ip', 'local_ip', 'servers.status', 'farm_roles.alias'), $args);
foreach ($response["data"] as &$row) {
try {
$dbServer = DBServer::LoadByID($row['server_id']);
$row['cloud_server_id'] = $dbServer->GetCloudServerID();
if (in_array($dbServer->status, array(SERVER_STATUS::RUNNING, SERVER_STATUS::INIT))) {
$row['cluster_role'] = "";
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->getDbMsrBehavior() || $dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MYSQL)) {
$isMaster = $dbServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) || $dbServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER);
$row['cluster_role'] = $isMaster ? 'Master' : 'Slave';
if ($isMaster && $dbServer->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER) || $dbServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_MYSQL_SLAVE_TO_MASTER)) {
$row['cluster_role'] = 'Promoting';
}
}
}
$row['cloud_location'] = $dbServer->GetCloudLocation();
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
$loc = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::AVAIL_ZONE);
if ($loc && $loc != 'x-scalr-diff') {
$row['cloud_location'] .= "/" . substr($loc, -1, 1);
}
}
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
$row['has_eip'] = $this->db->GetOne("SELECT id FROM elastic_ips WHERE server_id = ?", array($dbServer->serverId));
}
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$shardIndex = $dbServer->GetProperty(Scalr_Role_Behavior_MongoDB::SERVER_SHARD_INDEX);
$replicaSetIndex = $dbServer->GetProperty(Scalr_Role_Behavior_MongoDB::SERVER_REPLICA_SET_INDEX);
$row['cluster_position'] = "{$shardIndex}-{$replicaSetIndex}";
}
} catch (Exception $e) {
}
$rebooting = $this->db->GetOne("SELECT value FROM server_properties WHERE server_id=? AND `name`=? LIMIT 1", array($row['server_id'], SERVER_PROPERTIES::REBOOTING));
if ($dbServer->status == SERVER_STATUS::RUNNING) {
if ($rebooting) {
$row['status'] = "Rebooting";
}
$subStatus = $dbServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS);
if ($subStatus) {
$row['status'] = ucfirst($subStatus);
}
}
$row['is_locked'] = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) ? 1 : 0;
$row['is_szr'] = $dbServer->IsSupported("0.5");
$row['initDetailsSupported'] = $dbServer->IsSupported("0.7.181");
if ($dbServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED) && in_array($dbServer->status, array(SERVER_STATUS::INIT, SERVER_STATUS::PENDING))) {
$row['isInitFailed'] = 1;
}
$launchError = $dbServer->GetProperty(SERVER_PROPERTIES::LAUNCH_ERROR);
if ($launchError) {
$row['launch_error'] = "1";
}
$serverAlerts = new Alerts($dbServer);
$row['agent_version'] = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_VESION);
$row['agent_update_needed'] = $dbServer->IsSupported("0.7") && !$dbServer->IsSupported("0.7.189");
$row['agent_update_manual'] = !$dbServer->IsSupported("0.5");
$row['os_family'] = $dbServer->GetOsFamily();
$row['flavor'] = $dbServer->GetFlavor();
$row['alerts'] = $serverAlerts->getActiveAlertsCount();
if (!$row['flavor']) {
$row['flavor'] = '';
}
if ($dbServer->status == SERVER_STATUS::RUNNING) {
$tm = (int) $dbServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if (!$tm) {
$tm = (int) strtotime($row['dtadded']);
}
if ($tm > 0) {
$row['uptime'] = Scalr_Util_DateTime::getHumanReadableTimeout(time() - $tm, false);
}
} else {
$row['uptime'] = '';
}
$r_dns = $this->db->GetOne("SELECT value FROM farm_role_settings WHERE farm_roleid=? AND `name`=? LIMIT 1", array($row['farm_roleid'], DBFarmRole::SETTING_EXCLUDE_FROM_DNS));
$row['excluded_from_dns'] = !$dbServer->GetProperty(SERVER_PROPERTIES::EXCLUDE_FROM_DNS) && !$r_dns ? false : true;
}
$this->response->data($response);
}