/** * Checks if user has permissions to project in environment or account scope * * @param string $projectId Identifier of the project * @param array $criteria ['envId' => '', 'clientid' => ''] * @return bool|mixed */ public function checkPermission($projectId, array $criteria) { $and = ''; foreach ($criteria as $name => $value) { $field = 'f.' . \Scalr::decamelize($name); $and .= " AND " . $field . "=" . $this->db->escape($value); } $projectEntity = new ProjectEntity(); $projectId = $projectEntity->type('projectId')->toDb($projectId); $where = " WHERE p.project_id = UNHEX('" . $projectId . "') AND EXISTS (\n SELECT * FROM farms f\n LEFT JOIN farm_settings fs ON f.id = fs.farmid\n WHERE fs.name = '" . Entity\FarmSetting::PROJECT_ID . "'\n AND REPLACE(fs.value, '-', '') = HEX(p.project_id)\n {$and})"; $sql = "SELECT " . $projectEntity->fields('p') . "\n FROM " . $projectEntity->table('p') . $where; return $this->db->GetOne($sql); }
/** * Checks whether current connection is alive * * @param ADODB_mysqli $conn * @return bool Returns true on success or false otherwise */ private static function isConnectionAlive($conn) { $alive = true; if (!empty($conn->_connectionID) && method_exists($conn->_connectionID, 'ping')) { $alive = (bool) @$conn->_connectionID->ping(); } else { try { $conn->GetOne('SELECT 1'); } catch (\ADODB_Exception $e) { if (stristr($e->getMessage(), 'has gone away') !== false) { $alive = false; } } } return $alive; }
/** * @param Scalr_Service_ZohoCrm_Entity_Contact $contact * @param Client $client * @return void */ private function bindContact($contact, $client) { list($contact->firstName, $contact->lastName) = explode(" ", $client->Fullname, 2); if (!$contact->lastName) { $contact->lastName = $contact->firstName; unset($contact->firstName); } $contact->email = $client->Email; $contact->phone = $client->Phone; $contact->fax = $client->Fax; $contact->mailingStreet = $client->Address1; $contact->mailingCity = $client->City; $contact->mailingState = $client->State; $contact->mailingCode = $client->ZipCode; if ($client->Country) { $contact->mailingCountry = $this->db->GetOne("\n SELECT name FROM countries WHERE code = ?\n LIMIT 1\n ", array($client->Country)); } $adPagesVisited = $client->GetSettingValue(CLIENT_SETTINGS::AD_PAGES_VISITED); $adCompaign = $client->GetSettingValue(CLIENT_SETTINGS::AD_COMPAIGN); if ($adPagesVisited) { $contact->leadSource = 'Adwords'; $contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_AD_PAGES_VISITED, (int) $adPagesVisited); $contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_AD_VALUE_TRACK, $client->GetSettingValue(CLIENT_SETTINGS::AD_VALUE_TRACK)); $client->ClearSettings('adwords%'); } elseif ($adCompaign) { $contact->leadSource = $adCompaign; } else { $packageId = $client->GetSettingValue(CLIENT_SETTINGS::BILLING_PACKAGE); if (!$packageId || $packageId == 4) { $contact->leadSource = "Development edition"; } else { $contact->leadSource = "Production edition"; } } $unsubscrDate = $client->GetSettingValue(CLIENT_SETTINGS::ZOHOCRM_UNSUBSCR_DATE); $contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_UNSUBSCRIBED_ACCOUNT, (bool) $unsubscrDate); $contact->setProperty(Scalr_Integration_ZohoCrm_CustomFields::CONTACT_DATE_UNSUBSCRIBED, $unsubscrDate ? $unsubscrDate : null); }
protected function buildResponseFromSql($sql, $filterFields = array(), $groupSQL = "", $simpleQuery = true, $noLimit = false) { $this->request->defineParams(array('start' => array('type' => 'int', 'default' => 0), 'limit' => array('type' => 'int', 'default' => 20))); if (is_array($groupSQL)) { return $this->buildResponseFromSql2($sql, $filterFields, $groupSQL, is_array($simpleQuery) ? $simpleQuery : array(), $noLimit); } if ($this->getParam('query') && count($filterFields) > 0) { $filter = $this->db->qstr('%' . trim($this->getParam('query')) . '%'); foreach ($filterFields as $field) { if ($simpleQuery) { $likes[] = "`{$field}` LIKE {$filter}"; } else { $likes[] = "{$field} LIKE {$filter}"; } } $sql .= " AND ("; $sql .= implode(" OR ", $likes); $sql .= ")"; } if ($groupSQL) { $sql .= "{$groupSQL}"; } if (!$noLimit) { $response['total'] = $this->db->GetOne('SELECT COUNT(*) FROM (' . $sql . ') c_sub'); } // @TODO replace with simple code (legacy code) $s = $this->getParam('sort'); if (!is_array($s)) { $s = json_decode($this->getParam('sort'), true); } if (is_array($s)) { $sorts = array(); if (count($s) && (!isset($s[0]) || !is_array($s[0]))) { $s = array($s); } foreach ($s as $param) { $sort = preg_replace("/[^A-Za-z0-9_]+/", "", $param['property']); $dir = in_array(strtolower($param['direction']), array('asc', 'desc')) ? $param['direction'] : 'ASC'; if ($sort && $dir) { $sorts[] = "`{$sort}` {$dir}"; } } if (count($sorts) > 0) { $sql .= " ORDER BY " . implode($sorts, ','); } } else { if ($this->getParam('sort')) { $sort = preg_replace("/[^A-Za-z0-9_]+/", "", $this->getParam('sort')); $dir = in_array(strtolower($this->getParam('dir')), array('asc', 'desc')) ? $this->getParam('dir') : 'ASC'; $sql .= " ORDER BY `{$sort}` {$dir}"; } } if (!$noLimit) { $start = intval($this->getParam('start')); if ($start > $response["total"]) { $start = 0; } $limit = intval($this->getParam('limit')); $sql .= " LIMIT {$start}, {$limit}"; } //$response['sql'] = $sql; $response["success"] = true; $response["data"] = $this->db->GetAll($sql); return $response; }
/** * Set roles for specified user for specified team. * * @param int $teamId The identifier of the team * @param int $userId The identifier of the user * @param array $accountRoles The list of the identifiers of the roles of account level * @param int $accountId optional The identifier of the account */ public function setUserRoles($teamId, $userId, $accountRoles, $accountId = null) { $accountId = intval($accountId); //Verify that team and user are from the same acount if (!empty($accountId)) { $check = $this->db->GetOne("\n SELECT 1 FROM account_users WHERE id = ? AND account_id = ? LIMIT 1\n ", array($userId, $accountId)) && $this->db->GetOne("\n SELECT 1 FROM account_teams WHERE id = ? AND account_id = ? LIMIT 1\n ", array($teamId, $accountId)); if (!$check) { throw new Exception\AclException(sprintf('Cannot find the team "%d" or user "%d" in the account "%d"', $teamId, $userId, $accountId)); } } else { //Retrieves identifier of the account $accountId = $this->db->GetOne("\n SELECT u.account_id\n FROM account_users u\n JOIN account_teams t ON t.account_id = u.account_id\n WHERE u.user_id = ? AND t.team_id = ?\n LIMIT 1\n ", array($userId, $accountId)); if (!$accountId) { throw new Exception\AclException(sprintf('Cannot find the team "%d" or user "%d" in the account "%d"', $teamId, $userId, $accountId)); } } $teamUserId = $this->db->GetOne("\n SELECT tu.id\n FROM `account_team_users` tu\n WHERE tu.`team_id` = ? AND tu.`user_id` = ?\n LIMIT 1\n ", array($teamId, $userId)); if (empty($teamUserId)) { $this->db->Execute("\n INSERT IGNORE `account_team_users`\n SET team_id = ?,\n user_id = ?\n ", array($teamId, $userId)); $teamUserId = $this->db->Insert_ID(); } else { //Removes previous relations $this->db->Execute("\n DELETE FROM `account_team_user_acls` WHERE account_team_user_id = ?\n ", array($teamUserId)); } if ($c = count($accountRoles)) { //Creates new relations $this->db->Execute("\n INSERT IGNORE `account_team_user_acls` (account_team_user_id, account_role_id)\n SELECT ?, r.account_role_id\n FROM `acl_account_roles` r\n WHERE r.account_id = ?\n AND r.account_role_id IN (" . rtrim(str_repeat("?,", $c), ',') . ")\n ", array_merge(array($teamUserId, $accountId), array_values($accountRoles))); } }
/** * Returns client setting value by name * * @param string $name * @return mixed $value */ public function GetSettingValue($name) { return $this->DB->GetOne("SELECT value FROM client_settings WHERE clientid=? AND `key`=? LIMIT 1", array($this->ID, $name)); }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $serverId = $request->serverId; $logger = Logger::getLogger(__CLASS__); $this->log("INFO", "Processing messages for %s server", $serverId); try { $dbserver = DBServer::LoadByID($serverId); if ($dbserver->farmId) { if ($dbserver->GetFarmObject()->Status == FARM_STATUS::TERMINATED) { throw new ServerNotFoundException("Farm related to this server has been terminated."); } } } catch (ServerNotFoundException $e) { //By some reason server does not exist $this->db->Execute("\n DELETE m FROM messages m\n WHERE m.server_id = ? AND m.`type` = ? AND m.`status` = ?\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]); return false; } //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); $rs = $this->db->Execute("\n SELECT m.* FROM messages m\n WHERE m.server_id = ? AND m.type = ? AND m.status = ?\n ORDER BY m.dtadded ASC\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]); while ($row = $rs->FetchRow()) { try { if ($row["message_format"] == 'xml') { $message = $this->serializer->unserialize($row["message"]); } else { $message = $this->jsonSerializer->unserialize($row["message"]); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_MESSAGE_FORMAT, 'json'); } $message->messageIpAddress = $row['ipaddress']; $event = null; $startTime = microtime(true); // Update scalarizr package version if ($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]) { $dbserver->setScalarizrVersion($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]); } if ($message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]) { $dbserver->SetProperty(SERVER_PROPERTIES::SZR_UPD_CLIENT_VERSION, $message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]); } if ($dbserver->GetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES)) { continue; } if ($message instanceof \Scalr_Messaging_Msg) { $this->log('INFO', "Handling '%s' for '%s' server", $message->getName(), $serverId); } try { if ($message instanceof Scalr_Messaging_Msg_OperationResult) { if ($message->status == 'ok' || $message->status == 'completed') { if ($message->name == 'Grow MySQL/Percona data volume' || $message->name == 'mysql.grow-volume') { $volumeConfig = $message->data ? $message->data : $message->result; $oldVolumeId = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::VOLUME_ID); $engine = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::DATA_STORAGE_ENGINE); try { $storageVolume = Scalr_Storage_Volume::init(); try { $storageVolume->loadById($volumeConfig->id); $storageVolume->setConfig($volumeConfig); $storageVolume->save(); } catch (Exception $e) { if (strpos($e->getMessage(), 'not found')) { $storageVolume->loadBy(array('id' => $volumeConfig->id, 'client_id' => $dbserver->clientId, 'env_id' => $dbserver->envId, 'name' => "'{$volumeConfig->tags->service}' data volume", 'type' => $engine, 'platform' => $dbserver->platform, 'size' => $volumeConfig->size, 'fstype' => $volumeConfig->fstype, 'purpose' => $volumeConfig->tags->service, 'farm_roleid' => $dbserver->farmRoleId, 'server_index' => $dbserver->index)); $storageVolume->setConfig($volumeConfig); $storageVolume->save(true); } else { throw $e; } } $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::VOLUME_ID, $volumeConfig->id, DBFarmRole::TYPE_LCL); if ($engine == MYSQL_STORAGE_ENGINE::EBS) { $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_EBS_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG); } elseif ($engine == MYSQL_STORAGE_ENGINE::RAID_EBS) { $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_RAID_DISK_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG); } // Remove old $storageVolume->delete($oldVolumeId); } catch (Exception $e) { Logger::getLogger(__CLASS__)->error(new FarmLogMessage($dbserver->farmId, "Cannot save storage volume: {$e->getMessage()}")); } } } elseif ($message->status == 'error' || $message->status == 'failed') { if ($message->name == 'Initialization' || $message->name == 'system.init') { $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED, 1); if (is_object($message->error)) { $errorText = $message->error->message; } elseif ($message->error) { $errorText = $message->error; } $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText); $event = new HostInitFailedEvent($dbserver, $errorText); } } } elseif ($message instanceof Scalr_Messaging_Msg_InitFailed) { $errorText = $message->reason; $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText); $event = new HostInitFailedEvent($dbserver, $errorText); } elseif ($message instanceof \Scalr_Messaging_Msg_RuntimeError) { $logger->fatal(new FarmLogMessage($dbserver->farmId, "Scalarizr failed to launch on server '{$dbserver->getNameByConvention()}' with runtime error: {$message->message}", $dbserver->serverId)); } elseif ($message instanceof Scalr_Messaging_Msg_UpdateControlPorts) { $apiPort = $message->api; $ctrlPort = $message->messaging; $snmpPort = $message->snmp; // Check API port; $currentApiPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_API_PORT); if (!$currentApiPort) { $currentApiPort = 8010; } if ($apiPort && $apiPort != $currentApiPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr API port was changed from {$currentApiPort} to {$apiPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_API_PORT, $apiPort); } // Check Control port $currentCtrlPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT); if (!$currentCtrlPort) { $currentCtrlPort = 8013; } if ($ctrlPort && $ctrlPort != $currentCtrlPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr Control port was changed from {$currentCtrlPort} to {$ctrlPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT, $ctrlPort); } //Check SNMP port $currentSnmpPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT); if (!$currentSnmpPort) { $currentSnmpPort = 8014; } if ($snmpPort && $snmpPort != $currentSnmpPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr SNMP port was changed from {$currentSnmpPort} to {$snmpPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, $snmpPort); } } elseif ($message instanceof Scalr_Messaging_Msg_Win_HostDown) { $status = PlatformFactory::NewPlatform($dbserver->platform)->GetServerRealStatus($dbserver); if ($status->isRunning()) { $event = new RebootBeginEvent($dbserver); } else { if ($dbserver->platform == SERVER_PLATFORMS::EC2) { if (!$status->isTerminated()) { //Stopping $logger->error(new FarmLogMessage($dbserver->farmId, "Server is in '{$status->getName()}' state. Ignoring HostDown event.", $dbserver->serverId)); $isStopping = true; } } if ($isStopping) { $dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $dbserver->remoteIp = ""; $dbserver->localIp = ""; $dbserver->status = SERVER_STATUS::SUSPENDED; $dbserver->Save(); } $event = new HostDownEvent($dbserver); $event->isSuspended = true; } } elseif ($message instanceof Scalr_Messaging_Msg_Win_PrepareBundleResult) { try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); } catch (Exception $e) { } if ($bundleTask) { if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::PREPARING) { if ($message->status == 'ok') { $metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'os' => $message->os, 'software' => $message->software); $bundleTask->setMetaData($metaData); $bundleTask->Save(); PlatformFactory::NewPlatform($bundleTask->platform)->CreateServerSnapshot($bundleTask); } else { $bundleTask->SnapshotCreationFailed("PrepareBundle procedure failed: {$message->lastError}"); } } } } elseif ($message instanceof Scalr_Messaging_Msg_DeployResult) { try { $deploymentTask = Scalr_Model::init(Scalr_Model::DM_DEPLOYMENT_TASK)->loadById($message->deployTaskId); } catch (Exception $e) { } if ($deploymentTask) { if ($message->status == 'error') { $deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_FAILED; $deploymentTask->lastError = $message->lastError; } else { $deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_DEPLOYED; $deploymentTask->dtDeployed = date("Y-m-d H:i:s"); } $deploymentTask->save(); } } elseif ($message instanceof Scalr_Messaging_Msg_Hello) { $event = $this->onHello($message, $dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_FireEvent) { //Validate event $isEventExist = $this->db->GetOne("\n SELECT id FROM event_definitions\n WHERE name = ? AND ((env_id = ? AND account_id = ?) OR (env_id IS NULL AND account_id = ?) OR (env_id IS NULL AND account_id IS NULL))\n LIMIT 1\n ", array($message->eventName, $dbserver->envId, $dbserver->clientId, $dbserver->clientId)); if ($isEventExist) { $event = new CustomEvent($dbserver, $message->eventName, (array) $message->params); } } elseif ($message instanceof Scalr_Messaging_Msg_HostUpdate) { try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_MongoDb) { /********* MONGODB *********/ try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_DbMsr) { /********* DBMSR *********/ try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_HostInit) { $event = $this->onHostInit($message, $dbserver); try { $dbserver->updateTimelog('ts_hi', $message->secondsSinceBoot, $message->secondsSinceStart); } catch (Exception $e) { } if (!$event) { continue; } } elseif ($message instanceof Scalr_Messaging_Msg_HostUp) { $event = $this->onHostUp($message, $dbserver); try { $dbserver->updateTimelog('ts_hu'); } catch (Exception $e) { } } elseif ($message instanceof Scalr_Messaging_Msg_HostDown) { $ignoreHostDown = false; $p = PlatformFactory::NewPlatform($dbserver->platform); $status = $p->GetServerRealStatus($dbserver); if ($dbserver->isOpenstack()) { if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) { $logger->error(new FarmLogMessage($dbserver->farmId, "Rackspace server is in " . $status->getName() . " state. Ignoring HostDown message.", $dbserver->serverId)); $isRebooting = true; } } elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) { if ($status->getName() == 'STOPPING') { // We don't know is this shutdown or stop so let's ignore HostDown // and wait for status change $doNotProcessMessage = true; $ignoreHostDown = true; } elseif ($status->getName() == 'RUNNING') { $isRebooting = true; } elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) { $isStopping = true; } } else { if ($p->getResumeStrategy() == 'init') { //TODO: Check is is stopping or shutting-down procedure. if (!$status->isTerminated()) { $isStopping = true; } } } if ($isStopping) { $dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $dbserver->remoteIp = ""; $dbserver->localIp = ""; $dbserver->status = SERVER_STATUS::SUSPENDED; $dbserver->Save(); $event = new HostDownEvent($dbserver); $event->isSuspended = true; } elseif ($isRebooting) { $event = new RebootBeginEvent($dbserver); } elseif (!$ignoreHostDown) { if ($dbserver->farmId) { $wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown')); if (!$wasHostDownFired) { $event = new HostDownEvent($dbserver); } } } } elseif ($message instanceof Scalr_Messaging_Msg_RebootStart) { $event = new RebootBeginEvent($dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_RebootFinish) { if (!$dbserver->localIp && $message->localIp) { $dbserver->localIp = $message->localIp; $dbserver->Save(); } $event = new RebootCompleteEvent($dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_BeforeHostUp) { $event = new BeforeHostUpEvent($dbserver); try { $dbserver->updateTimelog('ts_bhu'); } catch (Exception $e) { } } elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceAttached) { if ($dbserver->platform == SERVER_PLATFORMS::EC2) { $aws = $dbserver->GetEnvironmentObject()->aws($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION)); $instanceId = $dbserver->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID); //The main goal of using filters there is to considerably decrease the size of the response. $volumes = $aws->ec2->volume->describe(null, array(array('name' => VolumeFilterNameType::attachmentInstanceId(), 'value' => (string) $instanceId), array('name' => VolumeFilterNameType::attachmentDevice(), 'value' => (string) $message->deviceName), array('name' => VolumeFilterNameType::status(), 'value' => AMAZON_EBS_STATE::IN_USE))); foreach ($volumes as $volume) { /* @var $volume Scalr\Service\Aws\Ec2\DataType\VolumeData */ if ($volume->status == AMAZON_EBS_STATE::IN_USE && count($volume->attachmentSet) && $volume->attachmentSet[0]->instanceId == $instanceId && $volume->attachmentSet[0]->device == $message->deviceName) { $message->volumeId = $volume->volumeId; } } //Releases memory unset($volumes); $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } $event = new EBSVolumeAttachedEvent($dbserver, $message->deviceName, $message->volumeId); } elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceMounted) { // Single volume $ebsinfo = $this->db->GetRow("\n SELECT * FROM ec2_ebs WHERE volume_id=? LIMIT 1\n ", array($message->volumeId)); if ($ebsinfo) { $this->db->Execute("\n UPDATE ec2_ebs\n SET mount_status=?, isfsexist='1'\n WHERE id=?\n ", array(EC2_EBS_MOUNT_STATUS::MOUNTED, $ebsinfo['id'])); } $event = new EBSVolumeMountedEvent($dbserver, $message->mountpoint, $message->volumeId, $message->deviceName); } elseif ($message instanceof Scalr_Messaging_Msg_RebundleResult) { if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_OK) { $metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'dist' => $message->dist, 'os' => $message->os, 'software' => $message->software); if ($dbserver->platform == SERVER_PLATFORMS::EC2) { if ($message->aws) { if ($message->aws->rootDeviceType == 'ebs') { $tags[] = ROLE_TAGS::EC2_EBS; } if ($message->aws->virtualizationType == 'hvm') { $tags[] = ROLE_TAGS::EC2_HVM; } } else { $aws = $dbserver->GetEnvironmentObject()->aws($dbserver); try { $info = $aws->ec2->image->describe($dbserver->GetProperty(EC2_SERVER_PROPERTIES::AMIID))->get(0); if ($info->rootDeviceType == 'ebs') { $tags[] = ROLE_TAGS::EC2_EBS; } else { try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) { $tags[] = ROLE_TAGS::EC2_EBS; } } catch (Exception $e) { } } if ($info->virtualizationType == 'hvm') { $tags[] = ROLE_TAGS::EC2_HVM; } unset($info); } catch (Exception $e) { $metaData['tagsError'] = $e->getMessage(); try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) { $tags[] = ROLE_TAGS::EC2_EBS; } } catch (Exception $e) { } } //Releases memory $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } } $metaData['tags'] = $tags; $event = new RebundleCompleteEvent($dbserver, $message->snapshotId, $message->bundleTaskId, $metaData); } else { if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_FAILED) { $event = new RebundleFailedEvent($dbserver, $message->bundleTaskId, $message->lastError); } } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateDataBundleResult) { if ($message->status == "ok") { $event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE, array('snapshotConfig' => $message->snapshotConfig, 'logFile' => $message->logFile, 'logPos' => $message->logPos, 'dataBundleSize' => $message->dataBundleSize, 'snapshotId' => $message->snapshotId)); } else { $event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE); $event->lastError = $message->lastError; } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateBackupResult) { if ($message->status == "ok") { $event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP, array()); $event->backupParts = $message->backupParts; } else { $event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP); $event->lastError = $message->lastError; } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_PromoteToMasterResult) { $event = $this->onMysql_PromoteToMasterResult($message, $dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreatePmaUserResult) { $farmRole = DBFarmRole::LoadByID($message->farmRoleId); if ($message->status == "ok") { $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_USER, $message->pmaUser, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_PASS, $message->pmaPassword, DBFarmRole::TYPE_LCL); } else { $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_ERROR, $message->lastError, DBFarmRole::TYPE_LCL); } } elseif ($message instanceof Scalr_Messaging_Msg_RabbitMq_SetupControlPanelResult) { $farmRole = $dbserver->GetFarmRoleObject(); if ($message->status == "ok") { $mgmtHost = $dbserver->getSzrHost(); if ($message->port) { $mgmtURL = "http://{$mgmtHost}:{$message->port}/mgmt"; } elseif ($message->cpanelUrl) { $info = parse_url($message->cpanelUrl); $mgmtURL = "http://{$mgmtHost}:{$info['port']}/mgmt"; } $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, $dbserver->serverId, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_URL, $mgmtURL, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); } else { $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_ERROR_MSG, $message->lastError, DBFarmRole::TYPE_LCL); } } elseif ($message instanceof Scalr_Messaging_Msg_AmiScriptsMigrationResult) { try { //Open security group: if ($dbserver->platform == SERVER_PLATFORMS::EC2) { $info = PlatformFactory::NewPlatform($dbserver->platform)->GetServerExtendedInformation($dbserver); $sg = explode(", ", $info['Security groups']); foreach ($sg as $sgroup) { if ($sgroup != 'default') { // For Scalarizr $group_rules = array(array('rule' => 'tcp:8013:8013:0.0.0.0/0'), array('rule' => 'udp:8014:8014:0.0.0.0/0')); $aws = $dbserver->GetEnvironmentObject()->aws($dbserver); $ipPermissions = new \Scalr\Service\Aws\Ec2\DataType\IpPermissionList(); foreach ($group_rules as $rule) { $group_rule = explode(":", $rule["rule"]); $ipPermissions->append(new \Scalr\Service\Aws\Ec2\DataType\IpPermissionData($group_rule[0], $group_rule[1], $group_rule[2], new \Scalr\Service\Aws\Ec2\DataType\IpRangeData($group_rule[3]))); } $aws->ec2->securityGroup->authorizeIngress($ipPermissions, null, $sgroup); $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); unset($ipPermissions); break; } } } } catch (Exception $e) { $logger->fatal($e->getMessage()); } $dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, 8014); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, "0.7.217"); if ($message->mysql) { $event = $this->onHostUp($message, $dbserver, true); } } $handle_status = MESSAGE_STATUS::HANDLED; } catch (Exception $e) { $handle_status = MESSAGE_STATUS::FAILED; $logger->error(sprintf("Cannot handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s). %s", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId, $e->getMessage() . "({$e->getFile()}:{$e->getLine()})")); } if (!$doNotProcessMessage) { $totalTime = microtime(true) - $startTime; $this->db->Execute("\n UPDATE messages\n SET status = ?, processing_time = ?, dtlasthandleattempt = NOW()\n WHERE messageid = ?\n ", array($handle_status, $totalTime, $message->messageId)); } else { $logger->info(sprintf("Handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s) is postponed due to status transition", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId)); } if ($event) { \Scalr::FireEvent($dbserver->farmId, $event); } } catch (Exception $e) { $logger->error($e->getMessage()); } } return $request; }
/** * {@inheritdoc} * @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork() */ function handleWork($farmId) { $this->cleanup(); $DBFarm = DBFarm::LoadByID($farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; $this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})"); // Collect information from database $servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED)); $this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database"); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { return; } foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { /** @var DBServer $DBServer */ try { if ($DBServer->cloudLocation) { try { Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})")); $p = PlatformFactory::NewPlatform($DBServer->platform); $list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED")); } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage())); } if (!$serverInfo) { if ($p->debugLog) { Logger::getLogger('Openstack')->fatal($p->debugLog); } if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) { if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) { if (PlatformFactory::isOpenstack($DBServer->platform)) { $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1); } else { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform))); } continue; } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId)); } } else { $DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0"); $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0"); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); $env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false); return; } if (stristr($e->getMessage(), "Could not connect to host")) { continue; } print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; continue; } /* try { $realStatus = $DBServer->GetRealStatus()->getName(); if ($realStatus == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus); continue; } else { if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, ""); } } catch (Exception $e) {} */ try { if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform)) { if ($DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { // if server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]); $DBServer->remoteIp = ""; $DBServer->localIp = ""; $DBServer->status = SERVER_STATUS::SUSPENDED; $DBServer->Save(); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } else { // If server was suspended during initialization - we do not support this and need to terminate this instance $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { //TODO: Depends on resume strategy need to set server status // For Openstack we need to re-accociate IPs $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0); try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]); } } Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, "")); continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO: handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage())); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId)); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout))); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false); } catch (Exception $err) { $this->logger->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']))); Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO: Check health: } } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform); $this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId)); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; //TODO: $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() < $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO: Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { print $e->getTraceAsString() . "\n"; } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(10); print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } else { print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } } } }
/** * Checks if user is the lead of at least one project or cost center * * @param string $email User's email * @return bool Returns true if user is project or cc lead */ public function isLead($email) { $isLead = $this->db->GetOne("\n SELECT pp.project_id\n FROM project_properties pp\n JOIN projects p ON p.project_id = pp.project_id\n WHERE pp.name = ?\n AND pp.value = ?\n AND p.archived = ?\n\n UNION\n\n SELECT cp.cc_id\n FROM cc_properties cp\n JOIN ccs c ON c.cc_id = cp.cc_id\n WHERE cp.name = ?\n AND cp.value = ?\n AND c.archived = ?\n ", [ProjectPropertyEntity::NAME_LEAD_EMAIL, $email, ProjectEntity::NOT_ARCHIVED, CostCentrePropertyEntity::NAME_LEAD_EMAIL, $email, CostCentreEntity::NOT_ARCHIVED]); return $isLead ? true : false; }
function handleWork($farmId) { $DBFarm = DBFarm::LoadByID($farmId); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) { for ($i = 0; $i < 10; $i++) { if ($DBFarmRole->NewRoleID != '') { $this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled."); continue 2; } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) != '1' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { $this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping..."); continue 2; } // Get polling interval in seconds $polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60; $dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME); if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) { $this->logger->info("Polling interval: every {$polling_interval} seconds"); continue; } // Set Last polling time $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time(), DBFarmRole::TYPE_LCL); // Get current count of running and pending instances. $this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); $scalingDecision = $scalingManager->makeScalingDecition(); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue 2; } elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instance�s count after scaling resolution the spare instances are running�g for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; // Select instance that will be terminated // // * Instances ordered by uptime (oldest wil be choosen) // * Instance cannot be mysql master // * Choose the one that was rebundled recently while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING)))); if ($DBServer->index == 1 && $serversCount > 1) { continue; } } if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) { continue; } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { // We do not want to delete the most recently synced instance. Because of LA fluctuation. // I.e. LA may skyrocket during sync and drop dramatically after sync. if ($DBServer->dateLastSync != 0) { $chk_sync_time = $this->db->GetOne("\n SELECT server_id FROM servers\n WHERE dtlastsync > {$DBServer->dateLastSync}\n AND farm_roleid='{$DBServer->farmRoleId}'\n AND status NOT IN('" . SERVER_STATUS::TERMINATED . "', '" . SERVER_STATUS::TROUBLESHOOTING . "')\n LIMIT 1\n "); if ($chk_sync_time) { $got_valid_instance = true; } } else { $got_valid_instance = true; } } } if ($DBServer && $got_valid_instance) { $this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $aws = $DBServer->GetEnvironmentObject()->aws($DBServer); // Shutdown an instance just before a full hour running if (!$DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_IGNORE_FULL_HOUR)) { $response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0); if ($response && count($response->instancesSet)) { $launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp(); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')))); } } } else { $allow_terminate = true; } //Releases memory $DBServer->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } else { $allow_terminate = true; } if ($allow_terminate) { //Check safe shutdown if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) { if ($DBServer->IsSupported('0.11.3')) { try { $port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_API_PORT); if (!$port) { $port = 8010; } $szrClient = Scalr_Net_Scalarizr_Client::getClient($DBServer, Scalr_Net_Scalarizr_Client::NAMESPACE_SYSTEM, $port); $res = $szrClient->callAuthShutdownHook(); } catch (Exception $e) { $res = $e->getMessage(); } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled, but not supported by scalarizr installed on server '%s'. Ignoring.", $DBServer->serverId))); } if ($res != '1') { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res))); break; } } try { $DBServer->terminate('SCALING_DOWN', false); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { $this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage())); } } } else { $this->logger->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $DBFarm->ID, $DBFarmRole->ID)); } break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution �need more instances� for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count increase //Check DBMsr. Do not start slave during slave2master process $isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior(); if ($isDbMsr) { if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) { $runningServers = $DBFarmRole->GetRunningInstancesCount(); if ($runningServers > 0) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves"))); continue 2; } else { $DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL); } } } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name))); continue 2; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($DBFarm->ID)); if ($fstatus != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Scaling up"); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } } }
/** * {@inheritdoc} * @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork() */ function handleWork($serverId) { $dtNow = new DateTime('now'); $dbServer = DBServer::LoadByID($serverId); if ($dbServer->status != SERVER_STATUS::PENDING_TERMINATE && $dbServer->status != SERVER_STATUS::TERMINATED) { return; } if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) { try { if ($dbServer->GetCloudServerID()) { $serverHistory = $dbServer->getServerHistory(); if (!$dbServer->GetRealStatus()->isTerminated()) { try { if ($dbServer->farmId != 0) { if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED))); if ($dbServer->index == 1 && $serversCount > 1) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. " . "Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform))); return; } } Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform))); } } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->warn($serverId . ": {$e->getMessage()}"); } PlatformFactory::NewPlatform($dbServer->platform)->TerminateServer($dbServer); if ($dbServer->farmId) { $wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ?", array($serverId, 'HostDown')); if (!$wasHostDownFired) { Scalr::FireEvent($dbServer->farmId, new HostDownEvent($dbServer)); } } } else { if ($dbServer->status == SERVER_STATUS::TERMINATED) { if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) { $serverHistory->setTerminated(); $dbServer->Remove(); } } else { if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) { $dbServer->status = SERVER_STATUS::TERMINATED; $dbServer->Save(); } } } } else { //$serverHistory->setTerminated(); If there is no cloudserverID we don't need to add this server into server history. $dbServer->Remove(); } } catch (Exception $e) { if (stristr($e->getMessage(), "not found") || stristr($e->getMessage(), "could not be found") || stristr($e->getMessage(), "or entity does not exist or due to incorrect parameter annotation for the field in api cmd class")) { if ($serverHistory) { $serverHistory->setTerminated(); } $dbServer->Remove(); } else { throw $e; } } } }
/** * {@inheritdoc} * @see \Scalr\Upgrade\UpdateInterface::getTableDefinition() */ public function getTableDefinition($table, $schema = null) { if (!isset($schema)) { $schema = $this->db->GetOne("SELECT DATABASE()"); } $entity = new TableEntity(); $entity->db = $this->db; return $entity->findOne([['tableSchema' => $schema], ['tableName' => $table]]); }
private function onHostDown(\Scalr_Messaging_Msg $message, DBServer $dbserver) { // If insatnce is already SUSPENDED or TERMINATED it means that hostdown was already processed by CloudPoller // and no need to process it again if (in_array($dbserver->status, array(\SERVER_STATUS::SUSPENDED, \SERVER_STATUS::TERMINATED))) { return true; } $p = PlatformFactory::NewPlatform($dbserver->platform); $status = $p->GetServerRealStatus($dbserver); if ($dbserver->isOpenstack()) { $status = $p->GetServerRealStatus($dbserver); if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) { //Hard reboot $isRebooting = true; } elseif ($status->isRunning()) { // Soft reboot $isRebooting = true; } elseif (!$status->isTerminated()) { $isStopping = true; } } elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) { if ($status->getName() == 'STOPPING') { // We don't know is this shutdown or stop so let's ignore HostDown // and wait for status change return false; } elseif ($status->getName() == 'RUNNING') { $isRebooting = true; } elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) { $isStopping = true; } } else { if ($status->isRunning()) { $isRebooting = true; } elseif (!$status->isTerminated()) { $isStopping = true; } } if ($isStopping) { $event = new HostDownEvent($dbserver); $event->isSuspended = true; } elseif ($isRebooting) { $event = new RebootBeginEvent($dbserver); } else { if ($dbserver->farmId) { $wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown')); //TODO: if (!$wasHostDownFired) { $event = new HostDownEvent($dbserver); } } } return $event; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if (!isset($request->farmRoleId)) { //This is the farm with synchronous launch of roles try { $DBFarm = DBFarm::LoadByID($request->farmId); if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID); return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId); throw $e; } //Gets the list of the roles $list = $DBFarm->GetFarmRoles(); } else { //This is asynchronous lauhch try { $DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId); if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) { //We don't need to handle inactive farms return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId); throw $e; } $list = [$DBFarmRole]; } $this->getLogger()->debug("Processing %s FarmRoles", count($list)); foreach ($list as $DBFarmRole) { // Set Last polling time $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL); $disabledScaling = false; if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') { if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { // For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running) // When 1 instance is already running, the rest is fully manual $roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount(); if ($roleTotalInstances != 0) { $disabledScaling = true; } } else { if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { $disabledScaling = true; } } if ($disabledScaling) { $this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias); continue; } } $farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name; // Get current count of running and pending instances. $this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); //Replacing the logger $scalingManager->logger = $this->getLogger(); $scalingDecision = $scalingManager->makeScalingDecision(); $scalingDecisionDetails = $scalingManager->decisonInfo; $this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails)); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue; } else { if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instances count after scaling resolution the spare instances are running for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null)); continue; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; $ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR); $useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN); $isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ); // Select instance that will be terminated // // Instances ordered by uptime (oldest wil be choosen) // Instance cannot be mysql master // Choose the one that was rebundled recently $DBServer = null; while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($isRabbitMQ) { $serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]); if ($DBServer->index == 1 && $serverExists) { continue; } } if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) { continue; } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { $got_valid_instance = true; } //Check safe shutdown if ($useSafeShutdown == 1) { try { $res = $DBServer->scalarizr->system->callAuthShutdownHook(); } catch (Exception $e) { $res = $e->getMessage(); } if ($res != '1') { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); $got_valid_instance = false; } } } // end while if ($DBServer !== null && $got_valid_instance) { $this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $aws = $DBServer->GetEnvironmentObject()->aws($DBServer); // Shutdown an instance just before a full hour running if (!$ignoreFullHour) { $response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0); if ($response && count($response->instancesSet)) { $launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp(); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } } else { $allow_terminate = true; } //Releases memory $DBServer->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } else { $allow_terminate = true; } if ($allow_terminate) { $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } try { if ($terminateStrategy == 'terminate') { $DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } else { $DBServer->suspend('SCALING_DOWN', false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } catch (Exception $e) { $this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId)); } } } else { $this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID)); } //break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution 'need more instances' for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias)); continue; } } // end Timeout instance's count increase //Check DBMsr. Do not start slave during slave2master process $isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior(); if ($isDbMsr) { if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) { $runningServers = $DBFarmRole->GetRunningInstancesCount(); if ($runningServers > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves"))); continue; } else { $DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL); } } } if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName))); continue; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId)); if ($fstatus != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it."); return; } $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } $suspendedServer = null; if ($terminateStrategy == 'suspend') { $suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED)); if (count($suspendedServers) > 0) { $suspendedServer = array_shift($suspendedServers); } } if ($terminateStrategy == 'suspend' && $suspendedServer) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId)); } if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } catch (Exception $e) { \Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } else { $platform = PlatformFactory::NewPlatform($suspendedServer->platform); $platform->ResumeServer($suspendedServer); } } } } return true; }
function handleWork($farmId) { $this->cleanup(); $DBFarm = DBFarm::LoadByID($farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; $this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})"); // // Collect information from database // $servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED)); $this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database"); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { return; } foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { try { if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { $p = PlatformFactory::NewPlatform($DBServer->platform); if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] Crash check for server '{$DBServer->serverId}' failed: {$e->getMessage()}")); } if (!$serverInfo) { if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) { if ($DBServer->GetProperty("system.crashed") == 1) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } else { $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId))); $DBServer->SetProperty("system.crashed", "1"); } continue; } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] False-positive crash check: {$DBServer->serverId}")); if ($DBServer->platform == SERVER_PLATFORMS::EC2) { Logger::getLogger(LOG_CATEGORY::FARM)->fatal(sprintf("[CRASH][FarmID: {$DBFarm->ID}] InstanceID: %s, List: %s", $DBServer->GetCloudServerID(), json_encode($p->instancesListCache))); } } } else { $DBServer->SetProperty("system.crashed", "0"); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); $env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false); return; } if (stristr($e->getMessage(), "Could not connect to host")) { continue; } print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; continue; } try { $realStatus = $DBServer->GetRealStatus()->getName(); if ($realStatus == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus); continue; } else { if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, ""); } } } catch (Exception $e) { } try { if (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)) && $DBServer->GetRealStatus()->isTerminated()) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::TROUBLESHOOTING) { /* if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA) { if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED)) { $dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject(); $ssh2Client = new Scalr_Net_Ssh2_Client(); $ssh2Client->addPassword( $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER), $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS) ); $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT); if (!$port) $port = 22; try { $ssh2Client->connect($info['remoteIp'], $port); foreach ($DBServer->GetCloudUserData() as $k=>$v) $u_data .= "{$k}={$v};"; $u_data = trim($u_data, ";"); $ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false); $DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1); } catch(Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage())); } } } */ if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = Modules_Platforms_Ec2_Helpers_Eip::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } if ($DBServer->isOpenstack()) { $ipPool = $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_OPENSTACK_IP_POOL); if (!$DBServer->remoteIp && ($DBServer->status == SERVER_STATUS::PENDING || $DBServer->status == SERVER_STATUS::INIT) && $ipPool) { $osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION)); if ($osClient->hasService('network')) { $platform = PlatformFactory::NewPlatform($DBServer->platform); $serverIps = $platform->GetServerIPAddresses($DBServer); /********* USE Quantum (Neuron) NETWORK *******/ $ips = $osClient->network->floatingIps->list(); //Check free existing IP $ipAssigned = false; $ipAddress = false; $ipId = false; $ipInfo = false; foreach ($ips as $ip) { if ($ip->fixed_ip_address == $serverIps['localIp'] && $ip->port_id) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipAssigned = true; $ipInfo = $ip; break; } /* if (!$ip->fixed_ip_address && !$ipAddress) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipInfo = $ip; } */ } if ($ipInfo) { Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")")); } if (!$ipAssigned) { // Get instance port $ports = $osClient->network->ports->list(); foreach ($ports as $port) { if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $serverNetworkPort = $port->id; break; } } if (!$serverNetworkPort) { Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify network port of instance")); } else { if (!$ipAddress) { $publicNetworkId = $ipPool; /* $networks = $osClient->network->listNetworks(); foreach ($networks as $network) { if ($network->{"router:external"} == true) { $publicNetworkId = $network->id; } } if (!$publicNetworkId) { Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify public network to allocate" )); } else { */ $ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort); $ipAddress = $ip->floating_ip_address; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Allocated new IP {$ipAddress} for port: {$serverNetworkPort}")); //} } else { /* Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Updating IP {$ipAddress} ({$ipId}) with port: {$serverNetworkPort}" )); $osClient->network->floatingIps->update($ipId, $serverNetworkPort); */ } } } else { Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "IP: {$ipAddress} already assigned")); } } else { /********* USE NOVA NETWORK *******/ //Check free existing IP $ipAssigned = false; $ipAddress = false; $ips = $osClient->servers->floatingIps->list($ipPool); foreach ($ips as $ip) { //if (!$ip->instance_id) { // $ipAddress = $ip->ip; //} if ($ip->instance_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $ipAddress = $ip->ip; $ipAssigned = true; } } //If no free IP allocate new from pool if (!$ipAddress) { $ip = $osClient->servers->floatingIps->create($ipPool); $ipAddress = $ip->ip; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->ip, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); } if (!$ipAssigned) { //Associate floating IP with Instance $osClient->servers->addFloatingIp($DBServer->GetCloudServerID(), $ipAddress); } } if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); $DBServer->SetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES, null); } } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $platform = PlatformFactory::NewPlatform($DBServer->platform); $cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $DBServer->GetEnvironmentObject()), $DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobresult->jobstatus == 1) { $DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD, $res->jobresult->virtualmachine->password); $DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME, $res->jobresult->virtualmachine->name); } //TODO: handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage())); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate('FARM_ROLE_NOT_FOUND'); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId)); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout))); try { $DBServer->terminate(array('SERVER_DID_NOT_SEND_EVENT', $event, $launch_timeout), false); } catch (Exception $err) { $this->logger->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']))); Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO: Check health: } } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor()); $this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId)); } //Update GCE ServerID if ($DBServer->platform == SERVER_PLATFORMS::GCE) { if ($DBServer->GetProperty(GCE_SERVER_PROPERTIES::SERVER_ID) == $DBServer->serverId) { $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer); $DBServer->SetProperty(GCE_SERVER_PROPERTIES::SERVER_ID, $info['Cloud Server ID']); } } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; //TODO: $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isEnabled); } } else { //TODO: Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { print $e->getTraceAsString() . "\n"; } else { print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } } } }
/** * {@inheritdoc} * @see \Scalr\Upgrade\UpdateInterface::hasTableColumnDefault() */ public function hasTableColumnDefault($table, $column, $default, $schema = null) { $ret = $this->db->GetOne("\n SELECT 1 FROM `INFORMATION_SCHEMA`.`COLUMNS` s\n WHERE s.`TABLE_SCHEMA` = " . (isset($schema) ? $this->db->qstr($schema) : "DATABASE()") . "\n AND s.`TABLE_NAME` = " . $this->db->qstr($table) . "\n AND s.`COLUMN_NAME` = " . $this->db->qstr($column) . "\n AND s.`COLUMN_DEFAULT` " . ($default === null ? "IS NULL" : "=" . $this->db->qstr($default)) . "\n LIMIT 1\n "); return $ret ? true : false; }
/** * List all tables or tables which names match a RegExp * * @param string $match optional Regex to match tables * @return array List of matching tables */ public function showTables($match = null) { $tables = []; $stm = "SHOW TABLES"; if (!empty($match)) { $stm .= " WHERE Tables_in_" . $this->db->GetOne("SELECT DATABASE()") . " REGEXP " . $this->db->qstr($match); } $res = $this->db->Execute($stm); while ($row = $res->FetchRow()) { list(, $tables[]) = each($row); } return $tables; }