/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
try {
$dbFarm = DBFarm::LoadByID($farmId);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
return;
}
} catch (Exception $e) {
return;
}
foreach ($dbFarm->GetFarmRoles() as $dbFarmRole) {
$instancesHealth = array();
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($dbFarm->EnvID);
$aws = $env->aws($dbFarmRole);
$statusList = $aws->ec2->instance->describeStatus();
foreach ($statusList as $sd) {
/* @var $sd \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$instancesHealth[$sd->instanceId] = $sd;
}
unset($statusList);
//Releases memory
$env->getContainer()->release('aws');
unset($aws);
}
$servers = $this->db->Execute("\n SELECT server_id FROM servers\n WHERE farm_roleid = ? AND status = ?\n ", array($dbFarmRole->ID, SERVER_STATUS::RUNNING));
while ($server = $servers->FetchRow()) {
$dbServer = DBServer::LoadByID($server['server_id']);
// Do not support ami-scripts
if (!$dbServer->IsSupported("0.5")) {
continue;
}
// Do not support windows
if ($dbServer->IsSupported("0.8") && !$dbServer->IsSupported("0.9")) {
continue;
}
$subStatus = $dbServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS);
if ($subStatus != '') {
if ($subStatus == 'stopped') {
//Need to solve ALL failed metrics
$serverAlerts = new Alerts($dbServer);
if ($serverAlerts->getActiveAlertsCount()) {
$serverAlerts->solveAlert();
}
}
continue;
}
if ($dbServer->GetProperty(SERVER_PROPERTIES::REBOOTING)) {
continue;
}
$serverAlerts = new Alerts($dbServer);
//Check AWS healthchecks
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
try {
/* @var $statusInfo \Scalr\Service\Aws\Ec2\DataType\InstanceStatusData */
$statusInfo = isset($instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)]) ? $instancesHealth[$dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)] : null;
if ($statusInfo) {
$check = Alerts::METRIC_AWS_SYSTEM_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->systemStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($statusInfo->systemStatus->status != 'ok' && $statusInfo->systemStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->systemStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->systemStatus->status}: {$txtDetails}"));
}
$check = Alerts::METRIC_AWS_INSTANCE_STATUS;
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if ($statusInfo->instanceStatus->status == 'ok' && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} else {
if ($statusInfo->instanceStatus->status != 'ok' && $statusInfo->instanceStatus->status != 'initializing' && !$hasActiveAlert) {
$txtDetails = "";
$details = $statusInfo->instanceStatus->details;
if ($details) {
foreach ($details as $d) {
/* @var $d \Scalr\Service\Aws\Ec2\DataType\InstanceStatusDetailsSetData */
$txtDetails .= " {$d->name} is {$d->status},";
}
$txtDetails = trim($txtDetails, " ,");
unset($details);
}
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, "{$statusInfo->instanceStatus->status}: {$txtDetails}"));
}
}
unset($statusInfo);
}
} catch (Exception $e) {
}
}
//Not supported by VPC yet.
if ($dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
continue;
}
//Check scalr-upd-client status
$check = Alerts::METRIC_SCALARIZR_UPD_CLIENT_CONNECTIVITY;
$port = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_UPDC_PORT);
if (!$port) {
$port = 8008;
}
$result = $this->checkPort($dbServer->remoteIp, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
//Check scalarizr connectivity status
$check = Alerts::METRIC_SCALARIZR_CONNECTIVITY;
$port = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT);
if (!$port) {
$port = 8013;
}
$result = $this->checkPort($dbServer->remoteIp, $port);
$hasActiveAlert = $serverAlerts->hasActiveAlert($check);
if (!$result['status'] && !$hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckFailedEvent($dbServer, $check, $result['error']));
} elseif ($result['status'] && $hasActiveAlert) {
Scalr::FireEvent($dbServer->farmId, new CheckRecoveredEvent($dbServer, $check));
} elseif ($hasActiveAlert) {
$serverAlerts->updateLastCheckTime($check);
}
}
}
exit;
}
public function xListServersAction()
{
if (!$this->request->isAllowed(Acl::RESOURCE_FARMS_SERVERS) && !$this->request->isAllowed(Acl::RESOURCE_FARMS_ROLES, Acl::PERM_FARMS_ROLES_CREATE)) {
throw new Scalr_Exception_InsufficientPermissions();
}
$this->request->defineParams(array('roleId' => array('type' => 'int'), 'farmId' => array('type' => 'int'), 'farmRoleId' => array('type' => 'int'), 'serverId', 'hideTerminated' => array('type' => 'bool'), 'sort' => array('type' => 'json')));
$sql = 'SELECT servers.*, farms.name AS farm_name, roles.name AS role_name, farm_roles.alias AS role_alias
FROM servers
LEFT JOIN farms ON servers.farm_id = farms.id
LEFT JOIN roles ON roles.id = servers.role_id
LEFT JOIN farm_roles ON farm_roles.id = servers.farm_roleid
WHERE servers.env_id = ? AND :FILTER:';
$args = array($this->getEnvironmentId());
if ($this->getParam('cloudServerId')) {
$sql = str_replace('WHERE', 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id WHERE', $sql);
$sql .= ' AND (';
$sql .= 'server_properties.name = ? AND server_properties.value = ?';
$args[] = CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EC2_SERVER_PROPERTIES::INSTANCE_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EUCA_SERVER_PROPERTIES::INSTANCE_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = GCE_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = OPENSTACK_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = RACKSPACE_SERVER_PROPERTIES::SERVER_ID;
$args[] = $this->getParam('cloudServerId');
$sql .= ')';
}
if ($this->getParam('cloudServerLocation')) {
if (!strstr($sql, 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id')) {
$sql = str_replace('WHERE', 'LEFT JOIN server_properties ON servers.server_id = server_properties.server_id WHERE', $sql);
}
$sql .= ' AND (';
$sql .= 'server_properties.name = ? AND server_properties.value = ?';
$args[] = CLOUDSTACK_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EC2_SERVER_PROPERTIES::REGION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = EUCA_SERVER_PROPERTIES::REGION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = GCE_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ' OR server_properties.name = ? AND server_properties.value = ?';
$args[] = RACKSPACE_SERVER_PROPERTIES::DATACENTER;
$args[] = $this->getParam('cloudServerLocation');
$sql .= ')';
}
if ($this->getParam('farmId')) {
$sql .= " AND farm_id=?";
$args[] = $this->getParam('farmId');
}
if ($this->request->isAllowed(Acl::RESOURCE_FARMS_SERVERS)) {
if (!$this->request->isAllowed(Acl::RESOURCE_FARMS, Acl::PERM_FARMS_NOT_OWNED_FARMS)) {
$sql .= " AND (farms.created_by_id = ? OR servers.status IN (?, ?) AND farms.id IS NULL)";
$args[] = $this->user->getId();
$args[] = SERVER_STATUS::IMPORTING;
$args[] = SERVER_STATUS::TEMPORARY;
}
} else {
//show servers related to role creation process only
$sql .= ' AND servers.status IN (?, ?)';
$args[] = SERVER_STATUS::IMPORTING;
$args[] = SERVER_STATUS::TEMPORARY;
}
if ($this->getParam('farmRoleId')) {
$sql .= " AND farm_roleid=?";
$args[] = $this->getParam('farmRoleId');
}
if ($this->getParam('roleId')) {
$sql .= " AND role_id=?";
$args[] = $this->getParam('roleId');
}
if ($this->getParam('serverId')) {
$sql .= " AND server_id=?";
$args[] = $this->getParam('serverId');
}
if ($this->getParam('hideTerminated')) {
$sql .= ' AND servers.status != ?';
$args[] = SERVER_STATUS::TERMINATED;
}
$response = $this->buildResponseFromSql2($sql, array('platform', 'farm_name', 'role_name', 'role_alias', 'index', 'server_id', 'remote_ip', 'local_ip', 'uptime', 'status'), array('servers.server_id', 'farm_id', 'farms.name', 'remote_ip', 'local_ip', 'servers.status', 'farm_roles.alias'), $args);
foreach ($response["data"] as &$row) {
try {
$dbServer = DBServer::LoadByID($row['server_id']);
$row['cloud_server_id'] = $dbServer->GetCloudServerID();
if (in_array($dbServer->status, array(SERVER_STATUS::RUNNING, SERVER_STATUS::INIT))) {
$row['cluster_role'] = "";
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->getDbMsrBehavior() || $dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MYSQL)) {
$isMaster = $dbServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) || $dbServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER);
$row['cluster_role'] = $isMaster ? 'Master' : 'Slave';
if ($isMaster && $dbServer->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER) || $dbServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_MYSQL_SLAVE_TO_MASTER)) {
$row['cluster_role'] = 'Promoting';
}
}
}
$row['cloud_location'] = $dbServer->GetCloudLocation();
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
$loc = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::AVAIL_ZONE);
if ($loc && $loc != 'x-scalr-diff') {
$row['cloud_location'] .= "/" . substr($loc, -1, 1);
}
}
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
$row['has_eip'] = $this->db->GetOne("SELECT id FROM elastic_ips WHERE server_id = ?", array($dbServer->serverId));
}
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$shardIndex = $dbServer->GetProperty(Scalr_Role_Behavior_MongoDB::SERVER_SHARD_INDEX);
$replicaSetIndex = $dbServer->GetProperty(Scalr_Role_Behavior_MongoDB::SERVER_REPLICA_SET_INDEX);
$row['cluster_position'] = "{$shardIndex}-{$replicaSetIndex}";
}
} catch (Exception $e) {
}
$rebooting = $this->db->GetOne("SELECT value FROM server_properties WHERE server_id=? AND `name`=? LIMIT 1", array($row['server_id'], SERVER_PROPERTIES::REBOOTING));
if ($dbServer->status == SERVER_STATUS::RUNNING) {
if ($rebooting) {
$row['status'] = "Rebooting";
}
$subStatus = $dbServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS);
if ($subStatus) {
$row['status'] = ucfirst($subStatus);
}
}
$row['is_locked'] = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) ? 1 : 0;
$row['is_szr'] = $dbServer->IsSupported("0.5");
$row['initDetailsSupported'] = $dbServer->IsSupported("0.7.181");
if ($dbServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED) && in_array($dbServer->status, array(SERVER_STATUS::INIT, SERVER_STATUS::PENDING))) {
$row['isInitFailed'] = 1;
}
$launchError = $dbServer->GetProperty(SERVER_PROPERTIES::LAUNCH_ERROR);
if ($launchError) {
$row['launch_error'] = "1";
}
$serverAlerts = new Alerts($dbServer);
$row['agent_version'] = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_VESION);
$row['agent_update_needed'] = $dbServer->IsSupported("0.7") && !$dbServer->IsSupported("0.7.189");
$row['agent_update_manual'] = !$dbServer->IsSupported("0.5");
$row['os_family'] = $dbServer->GetOsFamily();
$row['flavor'] = $dbServer->GetFlavor();
$row['alerts'] = $serverAlerts->getActiveAlertsCount();
if (!$row['flavor']) {
$row['flavor'] = '';
}
if ($dbServer->status == SERVER_STATUS::RUNNING) {
$tm = (int) $dbServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if (!$tm) {
$tm = (int) strtotime($row['dtadded']);
}
if ($tm > 0) {
$row['uptime'] = Scalr_Util_DateTime::getHumanReadableTimeout(time() - $tm, false);
}
} else {
$row['uptime'] = '';
}
$r_dns = $this->db->GetOne("SELECT value FROM farm_role_settings WHERE farm_roleid=? AND `name`=? LIMIT 1", array($row['farm_roleid'], DBFarmRole::SETTING_EXCLUDE_FROM_DNS));
$row['excluded_from_dns'] = !$dbServer->GetProperty(SERVER_PROPERTIES::EXCLUDE_FROM_DNS) && !$r_dns ? false : true;
}
$this->response->data($response);
}