/**
* Checks if user has permissions to project in environment or account scope
*
* @param string $projectId Identifier of the project
* @param array $criteria ['envId' => '', 'clientid' => '']
* @return bool|mixed
*/
public function checkPermission($projectId, array $criteria)
{
$and = '';
foreach ($criteria as $name => $value) {
$field = 'f.' . \Scalr::decamelize($name);
$and .= " AND " . $field . "=" . $this->db->escape($value);
}
$projectEntity = new ProjectEntity();
$projectId = $projectEntity->type('projectId')->toDb($projectId);
$where = " WHERE p.project_id = UNHEX('" . $projectId . "') AND EXISTS (\n SELECT * FROM farms f\n LEFT JOIN farm_settings fs ON f.id = fs.farmid\n WHERE fs.name = '" . Entity\FarmSetting::PROJECT_ID . "'\n AND REPLACE(fs.value, '-', '') = HEX(p.project_id)\n {$and})";
$sql = "SELECT " . $projectEntity->fields('p') . "\n FROM " . $projectEntity->table('p') . $where;
return $this->db->GetOne($sql);
}
/**
* Checks whether current connection is alive
*
* @param ADODB_mysqli $conn
* @return bool Returns true on success or false otherwise
*/
private static function isConnectionAlive($conn)
{
$alive = true;
if (!empty($conn->_connectionID) && method_exists($conn->_connectionID, 'ping')) {
$alive = (bool) @$conn->_connectionID->ping();
} else {
try {
$conn->GetOne('SELECT 1');
} catch (\ADODB_Exception $e) {
if (stristr($e->getMessage(), 'has gone away') !== false) {
$alive = false;
}
}
}
return $alive;
}
/**
* @param Scalr_Service_ZohoCrm_Entity_Contact $contact
* @param Client $client
* @return void
*/
private function bindContact($contact, $client)
{
list($contact->firstName, $contact->lastName) = explode(" ", $client->Fullname, 2);
if (!$contact->lastName) {
$contact->lastName = $contact->firstName;
unset($contact->firstName);
}
$contact->email = $client->Email;
$contact->phone = $client->Phone;
$contact->fax = $client->Fax;
$contact->mailingStreet = $client->Address1;
$contact->mailingCity = $client->City;
$contact->mailingState = $client->State;
$contact->mailingCode = $client->ZipCode;
if ($client->Country) {
$contact->mailingCountry = $this->db->GetOne("\n SELECT name FROM countries WHERE code = ?\n LIMIT 1\n ", array($client->Country));
}
$adPagesVisited = $client->GetSettingValue(CLIENT_SETTINGS::AD_PAGES_VISITED);
$adCompaign = $client->GetSettingValue(CLIENT_SETTINGS::AD_COMPAIGN);
if ($adPagesVisited) {
$contact->leadSource = 'Adwords';
$contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_AD_PAGES_VISITED, (int) $adPagesVisited);
$contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_AD_VALUE_TRACK, $client->GetSettingValue(CLIENT_SETTINGS::AD_VALUE_TRACK));
$client->ClearSettings('adwords%');
} elseif ($adCompaign) {
$contact->leadSource = $adCompaign;
} else {
$packageId = $client->GetSettingValue(CLIENT_SETTINGS::BILLING_PACKAGE);
if (!$packageId || $packageId == 4) {
$contact->leadSource = "Development edition";
} else {
$contact->leadSource = "Production edition";
}
}
$unsubscrDate = $client->GetSettingValue(CLIENT_SETTINGS::ZOHOCRM_UNSUBSCR_DATE);
$contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_UNSUBSCRIBED_ACCOUNT, (bool) $unsubscrDate);
$contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_DATE_UNSUBSCRIBED, $unsubscrDate ? $unsubscrDate : null);
}
protected function buildResponseFromSql($sql, $filterFields = array(), $groupSQL = "", $simpleQuery = true, $noLimit = false)
{
$this->request->defineParams(array('start' => array('type' => 'int', 'default' => 0), 'limit' => array('type' => 'int', 'default' => 20)));
if (is_array($groupSQL)) {
return $this->buildResponseFromSql2($sql, $filterFields, $groupSQL, is_array($simpleQuery) ? $simpleQuery : array(), $noLimit);
}
if ($this->getParam('query') && count($filterFields) > 0) {
$filter = $this->db->qstr('%' . trim($this->getParam('query')) . '%');
foreach ($filterFields as $field) {
if ($simpleQuery) {
$likes[] = "`{$field}` LIKE {$filter}";
} else {
$likes[] = "{$field} LIKE {$filter}";
}
}
$sql .= " AND (";
$sql .= implode(" OR ", $likes);
$sql .= ")";
}
if ($groupSQL) {
$sql .= "{$groupSQL}";
}
if (!$noLimit) {
$response['total'] = $this->db->GetOne('SELECT COUNT(*) FROM (' . $sql . ') c_sub');
}
// @TODO replace with simple code (legacy code)
$s = $this->getParam('sort');
if (!is_array($s)) {
$s = json_decode($this->getParam('sort'), true);
}
if (is_array($s)) {
$sorts = array();
if (count($s) && (!isset($s[0]) || !is_array($s[0]))) {
$s = array($s);
}
foreach ($s as $param) {
$sort = preg_replace("/[^A-Za-z0-9_]+/", "", $param['property']);
$dir = in_array(strtolower($param['direction']), array('asc', 'desc')) ? $param['direction'] : 'ASC';
if ($sort && $dir) {
$sorts[] = "`{$sort}` {$dir}";
}
}
if (count($sorts) > 0) {
$sql .= " ORDER BY " . implode($sorts, ',');
}
} else {
if ($this->getParam('sort')) {
$sort = preg_replace("/[^A-Za-z0-9_]+/", "", $this->getParam('sort'));
$dir = in_array(strtolower($this->getParam('dir')), array('asc', 'desc')) ? $this->getParam('dir') : 'ASC';
$sql .= " ORDER BY `{$sort}` {$dir}";
}
}
if (!$noLimit) {
$start = intval($this->getParam('start'));
if ($start > $response["total"]) {
$start = 0;
}
$limit = intval($this->getParam('limit'));
$sql .= " LIMIT {$start}, {$limit}";
}
//$response['sql'] = $sql;
$response["success"] = true;
$response["data"] = $this->db->GetAll($sql);
return $response;
}
/**
* Set roles for specified user for specified team.
*
* @param int $teamId The identifier of the team
* @param int $userId The identifier of the user
* @param array $accountRoles The list of the identifiers of the roles of account level
* @param int $accountId optional The identifier of the account
*/
public function setUserRoles($teamId, $userId, $accountRoles, $accountId = null)
{
$accountId = intval($accountId);
//Verify that team and user are from the same acount
if (!empty($accountId)) {
$check = $this->db->GetOne("\n SELECT 1 FROM account_users WHERE id = ? AND account_id = ? LIMIT 1\n ", array($userId, $accountId)) && $this->db->GetOne("\n SELECT 1 FROM account_teams WHERE id = ? AND account_id = ? LIMIT 1\n ", array($teamId, $accountId));
if (!$check) {
throw new Exception\AclException(sprintf('Cannot find the team "%d" or user "%d" in the account "%d"', $teamId, $userId, $accountId));
}
} else {
//Retrieves identifier of the account
$accountId = $this->db->GetOne("\n SELECT u.account_id\n FROM account_users u\n JOIN account_teams t ON t.account_id = u.account_id\n WHERE u.user_id = ? AND t.team_id = ?\n LIMIT 1\n ", array($userId, $accountId));
if (!$accountId) {
throw new Exception\AclException(sprintf('Cannot find the team "%d" or user "%d" in the account "%d"', $teamId, $userId, $accountId));
}
}
$teamUserId = $this->db->GetOne("\n SELECT tu.id\n FROM `account_team_users` tu\n WHERE tu.`team_id` = ? AND tu.`user_id` = ?\n LIMIT 1\n ", array($teamId, $userId));
if (empty($teamUserId)) {
$this->db->Execute("\n INSERT IGNORE `account_team_users`\n SET team_id = ?,\n user_id = ?\n ", array($teamId, $userId));
$teamUserId = $this->db->Insert_ID();
} else {
//Removes previous relations
$this->db->Execute("\n DELETE FROM `account_team_user_acls` WHERE account_team_user_id = ?\n ", array($teamUserId));
}
if ($c = count($accountRoles)) {
//Creates new relations
$this->db->Execute("\n INSERT IGNORE `account_team_user_acls` (account_team_user_id, account_role_id)\n SELECT ?, r.account_role_id\n FROM `acl_account_roles` r\n WHERE r.account_id = ?\n AND r.account_role_id IN (" . rtrim(str_repeat("?,", $c), ',') . ")\n ", array_merge(array($teamUserId, $accountId), array_values($accountRoles)));
}
}
/**
* Returns client setting value by name
*
* @param string $name
* @return mixed $value
*/
public function GetSettingValue($name)
{
return $this->DB->GetOne("SELECT value FROM client_settings WHERE clientid=? AND `key`=? LIMIT 1", array($this->ID, $name));
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$serverId = $request->serverId;
$logger = Logger::getLogger(__CLASS__);
$this->log("INFO", "Processing messages for %s server", $serverId);
try {
$dbserver = DBServer::LoadByID($serverId);
if ($dbserver->farmId) {
if ($dbserver->GetFarmObject()->Status == FARM_STATUS::TERMINATED) {
throw new ServerNotFoundException("Farm related to this server has been terminated.");
}
}
} catch (ServerNotFoundException $e) {
//By some reason server does not exist
$this->db->Execute("\n DELETE m FROM messages m\n WHERE m.server_id = ? AND m.`type` = ? AND m.`status` = ?\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]);
return false;
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
$rs = $this->db->Execute("\n SELECT m.* FROM messages m\n WHERE m.server_id = ? AND m.type = ? AND m.status = ?\n ORDER BY m.dtadded ASC\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]);
while ($row = $rs->FetchRow()) {
try {
if ($row["message_format"] == 'xml') {
$message = $this->serializer->unserialize($row["message"]);
} else {
$message = $this->jsonSerializer->unserialize($row["message"]);
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_MESSAGE_FORMAT, 'json');
}
$message->messageIpAddress = $row['ipaddress'];
$event = null;
$startTime = microtime(true);
// Update scalarizr package version
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]) {
$dbserver->setScalarizrVersion($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]);
}
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]) {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_UPD_CLIENT_VERSION, $message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]);
}
if ($dbserver->GetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES)) {
continue;
}
if ($message instanceof \Scalr_Messaging_Msg) {
$this->log('INFO', "Handling '%s' for '%s' server", $message->getName(), $serverId);
}
try {
if ($message instanceof Scalr_Messaging_Msg_OperationResult) {
if ($message->status == 'ok' || $message->status == 'completed') {
if ($message->name == 'Grow MySQL/Percona data volume' || $message->name == 'mysql.grow-volume') {
$volumeConfig = $message->data ? $message->data : $message->result;
$oldVolumeId = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::VOLUME_ID);
$engine = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::DATA_STORAGE_ENGINE);
try {
$storageVolume = Scalr_Storage_Volume::init();
try {
$storageVolume->loadById($volumeConfig->id);
$storageVolume->setConfig($volumeConfig);
$storageVolume->save();
} catch (Exception $e) {
if (strpos($e->getMessage(), 'not found')) {
$storageVolume->loadBy(array('id' => $volumeConfig->id, 'client_id' => $dbserver->clientId, 'env_id' => $dbserver->envId, 'name' => "'{$volumeConfig->tags->service}' data volume", 'type' => $engine, 'platform' => $dbserver->platform, 'size' => $volumeConfig->size, 'fstype' => $volumeConfig->fstype, 'purpose' => $volumeConfig->tags->service, 'farm_roleid' => $dbserver->farmRoleId, 'server_index' => $dbserver->index));
$storageVolume->setConfig($volumeConfig);
$storageVolume->save(true);
} else {
throw $e;
}
}
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::VOLUME_ID, $volumeConfig->id, DBFarmRole::TYPE_LCL);
if ($engine == MYSQL_STORAGE_ENGINE::EBS) {
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_EBS_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG);
} elseif ($engine == MYSQL_STORAGE_ENGINE::RAID_EBS) {
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_RAID_DISK_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG);
}
// Remove old
$storageVolume->delete($oldVolumeId);
} catch (Exception $e) {
Logger::getLogger(__CLASS__)->error(new FarmLogMessage($dbserver->farmId, "Cannot save storage volume: {$e->getMessage()}"));
}
}
} elseif ($message->status == 'error' || $message->status == 'failed') {
if ($message->name == 'Initialization' || $message->name == 'system.init') {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED, 1);
if (is_object($message->error)) {
$errorText = $message->error->message;
} elseif ($message->error) {
$errorText = $message->error;
}
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText);
$event = new HostInitFailedEvent($dbserver, $errorText);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_InitFailed) {
$errorText = $message->reason;
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText);
$event = new HostInitFailedEvent($dbserver, $errorText);
} elseif ($message instanceof \Scalr_Messaging_Msg_RuntimeError) {
$logger->fatal(new FarmLogMessage($dbserver->farmId, "Scalarizr failed to launch on server '{$dbserver->getNameByConvention()}' with runtime error: {$message->message}", $dbserver->serverId));
} elseif ($message instanceof Scalr_Messaging_Msg_UpdateControlPorts) {
$apiPort = $message->api;
$ctrlPort = $message->messaging;
$snmpPort = $message->snmp;
// Check API port;
$currentApiPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$currentApiPort) {
$currentApiPort = 8010;
}
if ($apiPort && $apiPort != $currentApiPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr API port was changed from {$currentApiPort} to {$apiPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_API_PORT, $apiPort);
}
// Check Control port
$currentCtrlPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT);
if (!$currentCtrlPort) {
$currentCtrlPort = 8013;
}
if ($ctrlPort && $ctrlPort != $currentCtrlPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr Control port was changed from {$currentCtrlPort} to {$ctrlPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT, $ctrlPort);
}
//Check SNMP port
$currentSnmpPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT);
if (!$currentSnmpPort) {
$currentSnmpPort = 8014;
}
if ($snmpPort && $snmpPort != $currentSnmpPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr SNMP port was changed from {$currentSnmpPort} to {$snmpPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, $snmpPort);
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_HostDown) {
$status = PlatformFactory::NewPlatform($dbserver->platform)->GetServerRealStatus($dbserver);
if ($status->isRunning()) {
$event = new RebootBeginEvent($dbserver);
} else {
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
if (!$status->isTerminated()) {
//Stopping
$logger->error(new FarmLogMessage($dbserver->farmId, "Server is in '{$status->getName()}' state. Ignoring HostDown event.", $dbserver->serverId));
$isStopping = true;
}
}
if ($isStopping) {
$dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$dbserver->remoteIp = "";
$dbserver->localIp = "";
$dbserver->status = SERVER_STATUS::SUSPENDED;
$dbserver->Save();
}
$event = new HostDownEvent($dbserver);
$event->isSuspended = true;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_PrepareBundleResult) {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
} catch (Exception $e) {
}
if ($bundleTask) {
if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::PREPARING) {
if ($message->status == 'ok') {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'os' => $message->os, 'software' => $message->software);
$bundleTask->setMetaData($metaData);
$bundleTask->Save();
PlatformFactory::NewPlatform($bundleTask->platform)->CreateServerSnapshot($bundleTask);
} else {
$bundleTask->SnapshotCreationFailed("PrepareBundle procedure failed: {$message->lastError}");
}
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DeployResult) {
try {
$deploymentTask = Scalr_Model::init(Scalr_Model::DM_DEPLOYMENT_TASK)->loadById($message->deployTaskId);
} catch (Exception $e) {
}
if ($deploymentTask) {
if ($message->status == 'error') {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_FAILED;
$deploymentTask->lastError = $message->lastError;
} else {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_DEPLOYED;
$deploymentTask->dtDeployed = date("Y-m-d H:i:s");
}
$deploymentTask->save();
}
} elseif ($message instanceof Scalr_Messaging_Msg_Hello) {
$event = $this->onHello($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_FireEvent) {
//Validate event
$isEventExist = $this->db->GetOne("\n SELECT id FROM event_definitions\n WHERE name = ? AND ((env_id = ? AND account_id = ?) OR (env_id IS NULL AND account_id = ?) OR (env_id IS NULL AND account_id IS NULL))\n LIMIT 1\n ", array($message->eventName, $dbserver->envId, $dbserver->clientId, $dbserver->clientId));
if ($isEventExist) {
$event = new CustomEvent($dbserver, $message->eventName, (array) $message->params);
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostUpdate) {
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_MongoDb) {
/********* MONGODB *********/
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DbMsr) {
/********* DBMSR *********/
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostInit) {
$event = $this->onHostInit($message, $dbserver);
try {
$dbserver->updateTimelog('ts_hi', $message->secondsSinceBoot, $message->secondsSinceStart);
} catch (Exception $e) {
}
if (!$event) {
continue;
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostUp) {
$event = $this->onHostUp($message, $dbserver);
try {
$dbserver->updateTimelog('ts_hu');
} catch (Exception $e) {
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostDown) {
$ignoreHostDown = false;
$p = PlatformFactory::NewPlatform($dbserver->platform);
$status = $p->GetServerRealStatus($dbserver);
if ($dbserver->isOpenstack()) {
if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) {
$logger->error(new FarmLogMessage($dbserver->farmId, "Rackspace server is in " . $status->getName() . " state. Ignoring HostDown message.", $dbserver->serverId));
$isRebooting = true;
}
} elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) {
if ($status->getName() == 'STOPPING') {
// We don't know is this shutdown or stop so let's ignore HostDown
// and wait for status change
$doNotProcessMessage = true;
$ignoreHostDown = true;
} elseif ($status->getName() == 'RUNNING') {
$isRebooting = true;
} elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) {
$isStopping = true;
}
} else {
if ($p->getResumeStrategy() == 'init') {
//TODO: Check is is stopping or shutting-down procedure.
if (!$status->isTerminated()) {
$isStopping = true;
}
}
}
if ($isStopping) {
$dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$dbserver->remoteIp = "";
$dbserver->localIp = "";
$dbserver->status = SERVER_STATUS::SUSPENDED;
$dbserver->Save();
$event = new HostDownEvent($dbserver);
$event->isSuspended = true;
} elseif ($isRebooting) {
$event = new RebootBeginEvent($dbserver);
} elseif (!$ignoreHostDown) {
if ($dbserver->farmId) {
$wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbserver);
}
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_RebootStart) {
$event = new RebootBeginEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_RebootFinish) {
if (!$dbserver->localIp && $message->localIp) {
$dbserver->localIp = $message->localIp;
$dbserver->Save();
}
$event = new RebootCompleteEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_BeforeHostUp) {
$event = new BeforeHostUpEvent($dbserver);
try {
$dbserver->updateTimelog('ts_bhu');
} catch (Exception $e) {
}
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceAttached) {
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION));
$instanceId = $dbserver->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID);
//The main goal of using filters there is to considerably decrease the size of the response.
$volumes = $aws->ec2->volume->describe(null, array(array('name' => VolumeFilterNameType::attachmentInstanceId(), 'value' => (string) $instanceId), array('name' => VolumeFilterNameType::attachmentDevice(), 'value' => (string) $message->deviceName), array('name' => VolumeFilterNameType::status(), 'value' => AMAZON_EBS_STATE::IN_USE)));
foreach ($volumes as $volume) {
/* @var $volume Scalr\Service\Aws\Ec2\DataType\VolumeData */
if ($volume->status == AMAZON_EBS_STATE::IN_USE && count($volume->attachmentSet) && $volume->attachmentSet[0]->instanceId == $instanceId && $volume->attachmentSet[0]->device == $message->deviceName) {
$message->volumeId = $volume->volumeId;
}
}
//Releases memory
unset($volumes);
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
}
$event = new EBSVolumeAttachedEvent($dbserver, $message->deviceName, $message->volumeId);
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceMounted) {
// Single volume
$ebsinfo = $this->db->GetRow("\n SELECT * FROM ec2_ebs WHERE volume_id=? LIMIT 1\n ", array($message->volumeId));
if ($ebsinfo) {
$this->db->Execute("\n UPDATE ec2_ebs\n SET mount_status=?, isfsexist='1'\n WHERE id=?\n ", array(EC2_EBS_MOUNT_STATUS::MOUNTED, $ebsinfo['id']));
}
$event = new EBSVolumeMountedEvent($dbserver, $message->mountpoint, $message->volumeId, $message->deviceName);
} elseif ($message instanceof Scalr_Messaging_Msg_RebundleResult) {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_OK) {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'dist' => $message->dist, 'os' => $message->os, 'software' => $message->software);
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
if ($message->aws) {
if ($message->aws->rootDeviceType == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
}
if ($message->aws->virtualizationType == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
} else {
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver);
try {
$info = $aws->ec2->image->describe($dbserver->GetProperty(EC2_SERVER_PROPERTIES::AMIID))->get(0);
if ($info->rootDeviceType == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
} else {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) {
$tags[] = ROLE_TAGS::EC2_EBS;
}
} catch (Exception $e) {
}
}
if ($info->virtualizationType == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
unset($info);
} catch (Exception $e) {
$metaData['tagsError'] = $e->getMessage();
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) {
$tags[] = ROLE_TAGS::EC2_EBS;
}
} catch (Exception $e) {
}
}
//Releases memory
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
}
}
$metaData['tags'] = $tags;
$event = new RebundleCompleteEvent($dbserver, $message->snapshotId, $message->bundleTaskId, $metaData);
} else {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_FAILED) {
$event = new RebundleFailedEvent($dbserver, $message->bundleTaskId, $message->lastError);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateDataBundleResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE, array('snapshotConfig' => $message->snapshotConfig, 'logFile' => $message->logFile, 'logPos' => $message->logPos, 'dataBundleSize' => $message->dataBundleSize, 'snapshotId' => $message->snapshotId));
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateBackupResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP, array());
$event->backupParts = $message->backupParts;
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_PromoteToMasterResult) {
$event = $this->onMysql_PromoteToMasterResult($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreatePmaUserResult) {
$farmRole = DBFarmRole::LoadByID($message->farmRoleId);
if ($message->status == "ok") {
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_USER, $message->pmaUser, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_PASS, $message->pmaPassword, DBFarmRole::TYPE_LCL);
} else {
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_ERROR, $message->lastError, DBFarmRole::TYPE_LCL);
}
} elseif ($message instanceof Scalr_Messaging_Msg_RabbitMq_SetupControlPanelResult) {
$farmRole = $dbserver->GetFarmRoleObject();
if ($message->status == "ok") {
$mgmtHost = $dbserver->getSzrHost();
if ($message->port) {
$mgmtURL = "http://{$mgmtHost}:{$message->port}/mgmt";
} elseif ($message->cpanelUrl) {
$info = parse_url($message->cpanelUrl);
$mgmtURL = "http://{$mgmtHost}:{$info['port']}/mgmt";
}
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, $dbserver->serverId, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_URL, $mgmtURL, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
} else {
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_ERROR_MSG, $message->lastError, DBFarmRole::TYPE_LCL);
}
} elseif ($message instanceof Scalr_Messaging_Msg_AmiScriptsMigrationResult) {
try {
//Open security group:
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$info = PlatformFactory::NewPlatform($dbserver->platform)->GetServerExtendedInformation($dbserver);
$sg = explode(", ", $info['Security groups']);
foreach ($sg as $sgroup) {
if ($sgroup != 'default') {
// For Scalarizr
$group_rules = array(array('rule' => 'tcp:8013:8013:0.0.0.0/0'), array('rule' => 'udp:8014:8014:0.0.0.0/0'));
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver);
$ipPermissions = new \Scalr\Service\Aws\Ec2\DataType\IpPermissionList();
foreach ($group_rules as $rule) {
$group_rule = explode(":", $rule["rule"]);
$ipPermissions->append(new \Scalr\Service\Aws\Ec2\DataType\IpPermissionData($group_rule[0], $group_rule[1], $group_rule[2], new \Scalr\Service\Aws\Ec2\DataType\IpRangeData($group_rule[3])));
}
$aws->ec2->securityGroup->authorizeIngress($ipPermissions, null, $sgroup);
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
unset($ipPermissions);
break;
}
}
}
} catch (Exception $e) {
$logger->fatal($e->getMessage());
}
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, 8014);
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, "0.7.217");
if ($message->mysql) {
$event = $this->onHostUp($message, $dbserver, true);
}
}
$handle_status = MESSAGE_STATUS::HANDLED;
} catch (Exception $e) {
$handle_status = MESSAGE_STATUS::FAILED;
$logger->error(sprintf("Cannot handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s). %s", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId, $e->getMessage() . "({$e->getFile()}:{$e->getLine()})"));
}
if (!$doNotProcessMessage) {
$totalTime = microtime(true) - $startTime;
$this->db->Execute("\n UPDATE messages\n SET status = ?, processing_time = ?, dtlasthandleattempt = NOW()\n WHERE messageid = ?\n ", array($handle_status, $totalTime, $message->messageId));
} else {
$logger->info(sprintf("Handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s) is postponed due to status transition", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId));
}
if ($event) {
\Scalr::FireEvent($dbserver->farmId, $event);
}
} catch (Exception $e) {
$logger->error($e->getMessage());
}
}
return $request;
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
// Collect information from database
$servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/** @var DBServer $DBServer */
try {
if ($DBServer->cloudLocation) {
try {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})"));
$p = PlatformFactory::NewPlatform($DBServer->platform);
$list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED"));
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
if (!$serverInfo) {
if ($p->debugLog) {
Logger::getLogger('Openstack')->fatal($p->debugLog);
}
if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
if (PlatformFactory::isOpenstack($DBServer->platform)) {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform)));
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId));
}
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0");
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
/*
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped')
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
} catch (Exception $e) {}
*/
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform)) {
if ($DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
// if server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} else {
// If server was suspended during initialization - we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
//TODO: Depends on resume strategy need to set server status
// For Openstack we need to re-accociate IPs
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(10);
print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
/**
* Checks if user is the lead of at least one project or cost center
*
* @param string $email User's email
* @return bool Returns true if user is project or cc lead
*/
public function isLead($email)
{
$isLead = $this->db->GetOne("\n SELECT pp.project_id\n FROM project_properties pp\n JOIN projects p ON p.project_id = pp.project_id\n WHERE pp.name = ?\n AND pp.value = ?\n AND p.archived = ?\n\n UNION\n\n SELECT cp.cc_id\n FROM cc_properties cp\n JOIN ccs c ON c.cc_id = cp.cc_id\n WHERE cp.name = ?\n AND cp.value = ?\n AND c.archived = ?\n ", [ProjectPropertyEntity::NAME_LEAD_EMAIL, $email, ProjectEntity::NOT_ARCHIVED, CostCentrePropertyEntity::NAME_LEAD_EMAIL, $email, CostCentreEntity::NOT_ARCHIVED]);
return $isLead ? true : false;
}
function handleWork($farmId)
{
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) {
for ($i = 0; $i < 10; $i++) {
if ($DBFarmRole->NewRoleID != '') {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled.");
continue 2;
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) != '1' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
$this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping...");
continue 2;
}
// Get polling interval in seconds
$polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60;
$dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME);
if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) {
$this->logger->info("Polling interval: every {$polling_interval} seconds");
continue;
}
// Set Last polling time
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time(), DBFarmRole::TYPE_LCL);
// Get current count of running and pending instances.
$this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue 2;
} elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instance�s count after scaling
resolution the spare instances are running�g for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
// Select instance that will be terminated
//
// * Instances ordered by uptime (oldest wil be choosen)
// * Instance cannot be mysql master
// * Choose the one that was rebundled recently
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))));
if ($DBServer->index == 1 && $serversCount > 1) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
// We do not want to delete the most recently synced instance. Because of LA fluctuation.
// I.e. LA may skyrocket during sync and drop dramatically after sync.
if ($DBServer->dateLastSync != 0) {
$chk_sync_time = $this->db->GetOne("\n SELECT server_id FROM servers\n WHERE dtlastsync > {$DBServer->dateLastSync}\n AND farm_roleid='{$DBServer->farmRoleId}'\n AND status NOT IN('" . SERVER_STATUS::TERMINATED . "', '" . SERVER_STATUS::TROUBLESHOOTING . "')\n LIMIT 1\n ");
if ($chk_sync_time) {
$got_valid_instance = true;
}
} else {
$got_valid_instance = true;
}
}
}
if ($DBServer && $got_valid_instance) {
$this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_IGNORE_FULL_HOUR)) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c'))));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
//Check safe shutdown
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) {
if ($DBServer->IsSupported('0.11.3')) {
try {
$port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$port) {
$port = 8010;
}
$szrClient = Scalr_Net_Scalarizr_Client::getClient($DBServer, Scalr_Net_Scalarizr_Client::NAMESPACE_SYSTEM, $port);
$res = $szrClient->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled, but not supported by scalarizr installed on server '%s'. Ignoring.", $DBServer->serverId)));
}
if ($res != '1') {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res)));
break;
}
}
try {
$DBServer->terminate('SCALING_DOWN', false);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
$this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
}
} else {
$this->logger->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $DBFarm->ID, $DBFarmRole->ID));
}
break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution �need more instances� for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue 2;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name)));
continue 2;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($DBFarm->ID));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Scaling up");
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($serverId)
{
$dtNow = new DateTime('now');
$dbServer = DBServer::LoadByID($serverId);
if ($dbServer->status != SERVER_STATUS::PENDING_TERMINATE && $dbServer->status != SERVER_STATUS::TERMINATED) {
return;
}
if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) {
try {
if ($dbServer->GetCloudServerID()) {
$serverHistory = $dbServer->getServerHistory();
if (!$dbServer->GetRealStatus()->isTerminated()) {
try {
if ($dbServer->farmId != 0) {
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED)));
if ($dbServer->index == 1 && $serversCount > 1) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. " . "Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform)));
return;
}
}
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform)));
}
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn($serverId . ": {$e->getMessage()}");
}
PlatformFactory::NewPlatform($dbServer->platform)->TerminateServer($dbServer);
if ($dbServer->farmId) {
$wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ?", array($serverId, 'HostDown'));
if (!$wasHostDownFired) {
Scalr::FireEvent($dbServer->farmId, new HostDownEvent($dbServer));
}
}
} else {
if ($dbServer->status == SERVER_STATUS::TERMINATED) {
if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) {
$serverHistory->setTerminated();
$dbServer->Remove();
}
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) {
$dbServer->status = SERVER_STATUS::TERMINATED;
$dbServer->Save();
}
}
}
} else {
//$serverHistory->setTerminated(); If there is no cloudserverID we don't need to add this server into server history.
$dbServer->Remove();
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found") || stristr($e->getMessage(), "could not be found") || stristr($e->getMessage(), "or entity does not exist or due to incorrect parameter annotation for the field in api cmd class")) {
if ($serverHistory) {
$serverHistory->setTerminated();
}
$dbServer->Remove();
} else {
throw $e;
}
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\Upgrade\UpdateInterface::getTableDefinition()
*/
public function getTableDefinition($table, $schema = null)
{
if (!isset($schema)) {
$schema = $this->db->GetOne("SELECT DATABASE()");
}
$entity = new TableEntity();
$entity->db = $this->db;
return $entity->findOne([['tableSchema' => $schema], ['tableName' => $table]]);
}
private function onHostDown(\Scalr_Messaging_Msg $message, DBServer $dbserver)
{
// If insatnce is already SUSPENDED or TERMINATED it means that hostdown was already processed by CloudPoller
// and no need to process it again
if (in_array($dbserver->status, array(\SERVER_STATUS::SUSPENDED, \SERVER_STATUS::TERMINATED))) {
return true;
}
$p = PlatformFactory::NewPlatform($dbserver->platform);
$status = $p->GetServerRealStatus($dbserver);
if ($dbserver->isOpenstack()) {
$status = $p->GetServerRealStatus($dbserver);
if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) {
//Hard reboot
$isRebooting = true;
} elseif ($status->isRunning()) {
// Soft reboot
$isRebooting = true;
} elseif (!$status->isTerminated()) {
$isStopping = true;
}
} elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) {
if ($status->getName() == 'STOPPING') {
// We don't know is this shutdown or stop so let's ignore HostDown
// and wait for status change
return false;
} elseif ($status->getName() == 'RUNNING') {
$isRebooting = true;
} elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) {
$isStopping = true;
}
} else {
if ($status->isRunning()) {
$isRebooting = true;
} elseif (!$status->isTerminated()) {
$isStopping = true;
}
}
if ($isStopping) {
$event = new HostDownEvent($dbserver);
$event->isSuspended = true;
} elseif ($isRebooting) {
$event = new RebootBeginEvent($dbserver);
} else {
if ($dbserver->farmId) {
$wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown'));
//TODO:
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbserver);
}
}
}
return $event;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (!isset($request->farmRoleId)) {
//This is the farm with synchronous launch of roles
try {
$DBFarm = DBFarm::LoadByID($request->farmId);
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID);
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId);
throw $e;
}
//Gets the list of the roles
$list = $DBFarm->GetFarmRoles();
} else {
//This is asynchronous lauhch
try {
$DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId);
if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) {
//We don't need to handle inactive farms
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId);
throw $e;
}
$list = [$DBFarmRole];
}
$this->getLogger()->debug("Processing %s FarmRoles", count($list));
foreach ($list as $DBFarmRole) {
// Set Last polling time
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$disabledScaling = false;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') {
if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
// For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running)
// When 1 instance is already running, the rest is fully manual
$roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount();
if ($roleTotalInstances != 0) {
$disabledScaling = true;
}
} else {
if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$disabledScaling = true;
}
}
if ($disabledScaling) {
$this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias);
continue;
}
}
$farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name;
// Get current count of running and pending instances.
$this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
//Replacing the logger
$scalingManager->logger = $this->getLogger();
$scalingDecision = $scalingManager->makeScalingDecision();
$scalingDecisionDetails = $scalingManager->decisonInfo;
$this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails));
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue;
} else {
if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instances count after scaling
resolution the spare instances are running for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null));
continue;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
$ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR);
$useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN);
$isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ);
// Select instance that will be terminated
//
// Instances ordered by uptime (oldest wil be choosen)
// Instance cannot be mysql master
// Choose the one that was rebundled recently
$DBServer = null;
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($isRabbitMQ) {
$serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]);
if ($DBServer->index == 1 && $serverExists) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
$got_valid_instance = true;
}
//Check safe shutdown
if ($useSafeShutdown == 1) {
try {
$res = $DBServer->scalarizr->system->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
if ($res != '1') {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
$got_valid_instance = false;
}
}
}
// end while
if ($DBServer !== null && $got_valid_instance) {
$this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$ignoreFullHour) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
try {
if ($terminateStrategy == 'terminate') {
$DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} else {
$DBServer->suspend('SCALING_DOWN', false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
} catch (Exception $e) {
$this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId));
}
}
} else {
$this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID));
}
//break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution 'need more instances' for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias));
continue;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName)));
continue;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it.");
return;
}
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
$suspendedServer = null;
if ($terminateStrategy == 'suspend') {
$suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED));
if (count($suspendedServers) > 0) {
$suspendedServer = array_shift($suspendedServers);
}
}
if ($terminateStrategy == 'suspend' && $suspendedServer) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId));
}
if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
} else {
$platform = PlatformFactory::NewPlatform($suspendedServer->platform);
$platform->ResumeServer($suspendedServer);
}
}
}
}
return true;
}
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
//
// Collect information from database
//
$servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
try {
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
$p = PlatformFactory::NewPlatform($DBServer->platform);
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] Crash check for server '{$DBServer->serverId}' failed: {$e->getMessage()}"));
}
if (!$serverInfo) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) {
if ($DBServer->GetProperty("system.crashed") == 1) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId)));
$DBServer->SetProperty("system.crashed", "1");
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] False-positive crash check: {$DBServer->serverId}"));
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
Logger::getLogger(LOG_CATEGORY::FARM)->fatal(sprintf("[CRASH][FarmID: {$DBFarm->ID}] InstanceID: %s, List: %s", $DBServer->GetCloudServerID(), json_encode($p->instancesListCache)));
}
}
} else {
$DBServer->SetProperty("system.crashed", "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
}
} catch (Exception $e) {
}
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)) && $DBServer->GetRealStatus()->isTerminated()) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::TROUBLESHOOTING) {
/*
if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA)
{
if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED))
{
$dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject();
$ssh2Client = new Scalr_Net_Ssh2_Client();
$ssh2Client->addPassword(
$dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER),
$dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS)
);
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT);
if (!$port) $port = 22;
try {
$ssh2Client->connect($info['remoteIp'], $port);
foreach ($DBServer->GetCloudUserData() as $k=>$v)
$u_data .= "{$k}={$v};";
$u_data = trim($u_data, ";");
$ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false);
$DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1);
}
catch(Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage()));
}
}
}
*/
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = Modules_Platforms_Ec2_Helpers_Eip::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
if ($DBServer->isOpenstack()) {
$ipPool = $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_OPENSTACK_IP_POOL);
if (!$DBServer->remoteIp && ($DBServer->status == SERVER_STATUS::PENDING || $DBServer->status == SERVER_STATUS::INIT) && $ipPool) {
$osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION));
if ($osClient->hasService('network')) {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$serverIps = $platform->GetServerIPAddresses($DBServer);
/********* USE Quantum (Neuron) NETWORK *******/
$ips = $osClient->network->floatingIps->list();
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ipId = false;
$ipInfo = false;
foreach ($ips as $ip) {
if ($ip->fixed_ip_address == $serverIps['localIp'] && $ip->port_id) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipAssigned = true;
$ipInfo = $ip;
break;
}
/*
if (!$ip->fixed_ip_address && !$ipAddress) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipInfo = $ip;
}
*/
}
if ($ipInfo) {
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")"));
}
if (!$ipAssigned) {
// Get instance port
$ports = $osClient->network->ports->list();
foreach ($ports as $port) {
if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$serverNetworkPort = $port->id;
break;
}
}
if (!$serverNetworkPort) {
Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify network port of instance"));
} else {
if (!$ipAddress) {
$publicNetworkId = $ipPool;
/*
$networks = $osClient->network->listNetworks();
foreach ($networks as $network) {
if ($network->{"router:external"} == true) {
$publicNetworkId = $network->id;
}
}
if (!$publicNetworkId) {
Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId,
"Unable to identify public network to allocate"
));
} else {
*/
$ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort);
$ipAddress = $ip->floating_ip_address;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Allocated new IP {$ipAddress} for port: {$serverNetworkPort}"));
//}
} else {
/*
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId,
"Updating IP {$ipAddress} ({$ipId}) with port: {$serverNetworkPort}"
));
$osClient->network->floatingIps->update($ipId, $serverNetworkPort);
*/
}
}
} else {
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "IP: {$ipAddress} already assigned"));
}
} else {
/********* USE NOVA NETWORK *******/
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ips = $osClient->servers->floatingIps->list($ipPool);
foreach ($ips as $ip) {
//if (!$ip->instance_id) {
// $ipAddress = $ip->ip;
//}
if ($ip->instance_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$ipAddress = $ip->ip;
$ipAssigned = true;
}
}
//If no free IP allocate new from pool
if (!$ipAddress) {
$ip = $osClient->servers->floatingIps->create($ipPool);
$ipAddress = $ip->ip;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->ip, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
}
if (!$ipAssigned) {
//Associate floating IP with Instance
$osClient->servers->addFloatingIp($DBServer->GetCloudServerID(), $ipAddress);
}
}
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
$DBServer->SetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES, null);
}
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $DBServer->GetEnvironmentObject()), $DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobresult->jobstatus == 1) {
$DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD, $res->jobresult->virtualmachine->password);
$DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME, $res->jobresult->virtualmachine->name);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate('FARM_ROLE_NOT_FOUND');
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array('SERVER_DID_NOT_SEND_EVENT', $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor());
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
//Update GCE ServerID
if ($DBServer->platform == SERVER_PLATFORMS::GCE) {
if ($DBServer->GetProperty(GCE_SERVER_PROPERTIES::SERVER_ID) == $DBServer->serverId) {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
$DBServer->SetProperty(GCE_SERVER_PROPERTIES::SERVER_ID, $info['Cloud Server ID']);
}
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isEnabled);
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\Upgrade\UpdateInterface::hasTableColumnDefault()
*/
public function hasTableColumnDefault($table, $column, $default, $schema = null)
{
$ret = $this->db->GetOne("\n SELECT 1 FROM `INFORMATION_SCHEMA`.`COLUMNS` s\n WHERE s.`TABLE_SCHEMA` = " . (isset($schema) ? $this->db->qstr($schema) : "DATABASE()") . "\n AND s.`TABLE_NAME` = " . $this->db->qstr($table) . "\n AND s.`COLUMN_NAME` = " . $this->db->qstr($column) . "\n AND s.`COLUMN_DEFAULT` " . ($default === null ? "IS NULL" : "=" . $this->db->qstr($default)) . "\n LIMIT 1\n ");
return $ret ? true : false;
}
/**
* List all tables or tables which names match a RegExp
*
* @param string $match optional Regex to match tables
* @return array List of matching tables
*/
public function showTables($match = null)
{
$tables = [];
$stm = "SHOW TABLES";
if (!empty($match)) {
$stm .= " WHERE Tables_in_" . $this->db->GetOne("SELECT DATABASE()") . " REGEXP " . $this->db->qstr($match);
}
$res = $this->db->Execute($stm);
while ($row = $res->FetchRow()) {
list(, $tables[]) = each($row);
}
return $tables;
}
