Launches server
| public static LaunchServer ( ServerCreateInfo $ServerCreateInfo = null, DBServer $DBServer = null, boolean $delayed = false, integer | array $reason, Scalr_Account_User | integer $user = null ) : DBServer | null | ||
| $ServerCreateInfo | ServerCreateInfo | optional The server create info |
| $DBServer | DBServer | optional The DBServer object |
| $delayed | boolean | optional |
| $reason | integer | array | optional |
| $user | Scalr_Account_User | integer | optional The Scalr_Account_User object or its unique identifier |
| Результат | DBServer | null | Returns the DBServer object on cussess or null otherwise |
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::startForking()
*/
function startForking($workQueue)
{
// Reopen DB connection after daemonizing
$this->db = $this->getContainer()->adodb;
// Check that time has come to cleanup dead servers
$doCleanup = false;
sem_acquire($this->cleanupSem);
try {
if (time() - (int) $this->lastCleanup->get(0) >= $this->cleanupInterval) {
$doCleanup = true;
$this->lastCleanup->put(0, time());
}
} catch (Exception $e) {
sem_release($this->cleanupSem);
}
sem_release($this->cleanupSem);
if ($doCleanup) {
$this->logger->info("Cleanup dead servers");
try {
$importing_servers = $this->db->GetAll("\n SELECT server_id FROM servers\n WHERE status IN(?, ?)\n AND `dtadded` < NOW() - INTERVAL 1 DAY\n ", array(SERVER_STATUS::IMPORTING, SERVER_STATUS::TEMPORARY));
foreach ($importing_servers as $ts) {
$dbServer = DBServer::LoadByID($ts['server_id']);
if ($dbServer->status == SERVER_STATUS::TEMPORARY) {
try {
$dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
} catch (Exception $e) {
}
} else {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$dbServer->Remove();
}
}
}
$pending_launch_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=?", array(SERVER_STATUS::PENDING_LAUNCH));
try {
foreach ($pending_launch_servers as $ts) {
$DBServer = DBServer::LoadByID($ts['server_id']);
if ($DBServer->status == SERVER_STATUS::PENDING_LAUNCH) {
$account = Scalr_Account::init()->loadById($DBServer->clientId);
if ($account->status == Scalr_Account::STATUS_ACTIVE) {
Scalr::LaunchServer(null, $DBServer);
}
}
}
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Can't load server with ID #'%s'", $ts['server_id'], $e->getMessage()));
}
} catch (Exception $e) {
$this->logger->fatal("Poller::cleanup failed: {$e->getMessage()}");
}
}
}
public function xLaunchNewServerAction()
{
$dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId'));
$dbFarm = $dbFarmRole->GetFarmObject();
$this->user->getPermissions()->validate($dbFarmRole);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
throw new Exception("You can launch servers only on running farms");
}
$dbRole = $dbFarmRole->GetRoleObject();
if ($dbRole->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
throw new Exception("Manual launch of VPC Router insatnces is not allowed");
}
$pendingInstancesCount = $dbFarmRole->GetPendingInstancesCount();
$maxInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES);
$minInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES);
if ($maxInstances < $minInstances + 1) {
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $maxInstances + 1, DBFarmRole::TYPE_CFG);
$warnMsg = sprintf(_("Server count has been increased. Scalr will now request a new server from your cloud. Since the server count was already at the maximum set for this role, we increased the maximum by one."), $dbRole->name, $dbRole->name);
}
$runningInstancesCount = $dbFarmRole->GetRunningInstancesCount();
if ($runningInstancesCount + $pendingInstancesCount >= $minInstances) {
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $minInstances + 1, DBFarmRole::TYPE_CFG);
}
$serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
Scalr::LaunchServer($serverCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY, $this->user);
if ($warnMsg) {
$this->response->warning($warnMsg);
} else {
$this->response->success('Server successfully launched');
}
}
/**
* Launches new server
* @param int $farmRoleId
* @param bool $increaseMinInstances
* @param bool $needConfirmation
* @throws Scalr_Exception_Core
*/
public function xLaunchNewServerAction($farmRoleId, $increaseMinInstances = false, $needConfirmation = true)
{
$dbFarmRole = DBFarmRole::LoadByID($farmRoleId);
$dbFarm = $dbFarmRole->GetFarmObject();
$this->user->getPermissions()->validate($dbFarmRole);
$this->request->restrictFarmAccess($dbFarm, Acl::PERM_FARMS_SERVERS);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
throw new Scalr_Exception_Core('You can launch servers only on running farms');
}
$dbRole = $dbFarmRole->GetRoleObject();
if ($dbRole->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
throw new Scalr_Exception_Core('Manual launch of VPC Router insatnces is not allowed');
}
if ($dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) == 1) {
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingMetrics = $scalingManager->getFarmRoleMetrics();
$hasScalingMetrics = count($scalingMetrics) > 0;
$curInstances = $dbFarmRole->GetPendingInstancesCount() + $dbFarmRole->GetRunningInstancesCount();
$maxInstances = $dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_MAX_INSTANCES);
$minInstances = $dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_MIN_INSTANCES);
if ($needConfirmation) {
$res = ['showConfirmation' => true];
if ($maxInstances < $curInstances + 1) {
$res['showIncreaseMaxInstancesWarning'] = true;
$res['maxInstances'] = $maxInstances + 1;
}
if ($hasScalingMetrics && $curInstances >= $minInstances) {
$res['showIncreaseMinInstancesConfirm'] = true;
}
$this->response->data($res);
return;
} else {
if ($maxInstances < $curInstances + 1) {
$dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_MAX_INSTANCES, $maxInstances + 1, Entity\FarmRoleSetting::TYPE_CFG);
}
if ($increaseMinInstances && $hasScalingMetrics && $curInstances >= $minInstances) {
$dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_MIN_INSTANCES, $minInstances + 1, Entity\FarmRoleSetting::TYPE_CFG);
}
}
}
$serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
Scalr::LaunchServer($serverCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY, $this->user);
$this->response->success('Server successfully launched');
}
public function StartThread($bundle_task_info)
{
$db = Core::GetDBInstance();
// Reconfigure observers;
Scalr::ReconfigureObservers();
$BundleTask = BundleTask::LoadById($bundle_task_info['id']);
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
} catch (ServerNotFoundException $e) {
if (!$BundleTask->snapshotId) {
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$BundleTask->setDate('finished');
$BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId);
$BundleTask->Save();
return;
}
} catch (Exception $e) {
//$this->Logger->error($e->getMessage());
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
// IF server is in pensing state
$status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer);
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
if ($status->isPending()) {
$BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName()));
exit;
} elseif ($status->isTerminated()) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
break;
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$DBServer->remoteIp = $ips['remoteIp'];
$DBServer->localIp = $ips['locateIp'];
$DBServer->save();
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$BundleTask->save();
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$BundleTask->Log(sprintf(_("Initializing SSH2 session to the server")));
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
//Prepare script
$BundleTask->Log(sprintf(_("Uploading builder scripts...")));
$behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
$options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $DBServer->platform, 'behaviour' => trim(str_replace("base", "", $behaviors)), 'queryenv-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/query-env", 'messaging-p2p.producer-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/messaging");
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_NODE_NAME%", "\r\n"), array($DBServer->platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', '', '', '', '', '', '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
$BundleTask->Log(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$BundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog));
$BundleTask->SnapshotCreationFailed($msg);
/** Terminate server **/
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$DBServer->status = SERVER_STATUS::PENDING_TERMINATE;
$DBServer->save();
$BundleTask->Log(sprintf("Temporary server '%s' (%s) has been terminated", $DBServer->serverId, $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)));
} else {
$BundleTask->Log($msg);
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($BundleTask->platform);
$platformModule->CreateServerSnapshot($BundleTask);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage());
$BundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($BundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS:
$r_farm_roles = array();
$BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType));
try {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$BundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND new_role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
$update_farm_dns_zones = array();
$completed_roles = 0;
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) {
$BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
$completed_roles++;
} else {
$servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
if (count($servers) == 0) {
$DBFarmRole->RoleID = $DBFarmRole->NewRoleID;
$DBFarmRole->NewRoleID = null;
$DBFarmRole->Save();
$update_farm_dns_zones[$DBFarmRole->FarmID] = 1;
$completed_roles++;
} else {
$metaData = $BundleTask->getSnapshotDetails();
foreach ($servers as $server) {
try {
$DBServer = DBServer::LoadByID($server['server_id']);
} catch (Exception $e) {
//TODO:
continue;
}
if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) {
$DBServer->roleId = $BundleTask->roleId;
$DBServer->Save();
if ($metaData['noServersReplace']) {
$BundleTask->Log(sprintf("'Do not replace servers' option was checked. Server '%s' won't be replaced to new image.", $DBServer->serverId));
} else {
$BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, already has all modifications. No need to replace it.", $DBServer->serverId));
}
if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t\t\t\t\t\t", array(sprintf("Farm was in 'Synchronizing' state. Server terminated when bundling was completed. Bundle task #%s", $BundleTask->id), $DBServer->serverId));
}
} else {
if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?)", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE))) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID);
$nDBServer = Scalr::LaunchServer($ServerCreateInfo);
$nDBServer->replaceServerID = $DBServer->serverId;
$nDBServer->Save();
$BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId));
}
}
// if serverid != bundletask->serverID
}
// foreach server
}
// count($servers)
}
}
if ($completed_roles == count($r_farm_roles)) {
$BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Save();
}
try {
if (count($update_farm_dns_zones) != 0) {
foreach ($update_farm_dns_zones as $farm_id => $v) {
$dnsZones = DBDNSZone::loadByFarmId($farm_id);
foreach ($dnsZones as $dnsZone) {
if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) {
$dnsZone->updateSystemRecords();
$dnsZone->save();
}
}
}
}
} catch (Exception $e) {
$this->Logger->fatal("DNS ZONE: {$e->getMessage()}");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = DBRole::loadById($DBServer->roleId);
if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($DBServer) {
$new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer);
} else {
$new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
} else {
//TODO:
//$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})");
}
}
try {
$DBRole = DBRole::createFromBundleTask($BundleTask);
} catch (Exception $e) {
$BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
if ($DBServer->farmId) {
// Create DBFarmRole object
// TODO: create DBFarm role
}
//$DBServer->Delete();
}
} catch (Exception $e) {
}
} else {
try {
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS));
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
$DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
} else {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
$DBFarmRole = DBFarmRole::LoadByID($farm_role['id']);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
}
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS;
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
$BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
$BundleTask->Save();
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
}
break;
}
}
public function ServerLaunch($FarmRoleID, $IncreaseMaxInstances = false)
{
try {
$DBFarmRole = DBFarmRole::LoadByID($FarmRoleID);
$DBFarm = DBFarm::LoadByID($DBFarmRole->FarmID);
} catch (Exception $e) {
throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID));
}
if ($DBFarm->EnvID != $this->Environment->id) {
throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID));
}
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
throw new Exception(sprintf("Farm ID #%s is not running", $DBFarm->ID));
}
$this->user->getPermissions()->validate($DBFarm);
$this->restrictFarmAccess($DBFarm, Acl::PERM_FARMS_SERVERS);
$isSzr = true;
$n = $DBFarmRole->GetPendingInstancesCount();
if ($n >= 5 && !$isSzr) {
throw new Exception("There are {$n} pending instances. You cannot launch new instances while you have 5 pending ones.");
}
$response = $this->CreateInitialResponse();
$max_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES);
$min_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES);
if ($IncreaseMaxInstances) {
if ($max_instances < $min_instances + 1) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1, DBFarmRole::TYPE_CFG);
}
}
if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() >= $max_instances) {
if ($IncreaseMaxInstances) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1, DBFarmRole::TYPE_CFG);
} else {
throw new Exception("Max instances limit reached. Use 'IncreaseMaxInstances' parameter or increase max isntances settings in UI");
}
}
if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() == $min_instances) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $min_instances + 1, DBFarmRole::TYPE_CFG);
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY_API, $this->user);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Starting new instance (API). ServerID = %s.", $DBServer->serverId), $DBServer->serverId));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::API)->error($e->getMessage());
}
$response->ServerID = $DBServer->serverId;
$response->CloudServerID = $DBServer->GetCloudServerID();
return $response;
}
public function xLaunchNewServerAction()
{
$dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId'));
$dbFarm = $dbFarmRole->GetFarmObject();
$this->user->getPermissions()->validate($dbFarmRole);
if ($dbFarm->Status != FARM_STATUS::RUNNING) {
throw new Exception("You can launch servers only on running farms");
}
$dbRole = $dbFarmRole->GetRoleObject();
$isSzr = $dbFarmRole->GetRoleObject()->isSupported("0.5");
$pendingInstancesCount = $dbFarmRole->GetPendingInstancesCount();
if ($pendingInstancesCount >= 5 && !$isSzr) {
throw new Exception("There are {$pendingInstancesCount} pending instances. You cannot launch new instances while you have 5 pending ones.");
}
$maxInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES);
$minInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES);
if ($maxInstances < $minInstances + 1) {
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $maxInstances + 1);
$warnMsg = sprintf(_("Server successfully launched. The number of running %s instances is equal to maximum instances setting for this role. Maximum Instances setting for role %s has been increased automatically"), $dbRole->name, $dbRole->name);
}
$runningInstancesCount = $dbFarmRole->GetRunningInstancesCount();
if ($runningInstancesCount + $pendingInstancesCount >= $minInstances) {
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $minInstances + 1);
}
$serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
Scalr::LaunchServer($serverCreateInfo);
if ($warnMsg) {
$this->response->warning($warnMsg);
} else {
$this->response->success('Server successfully launched');
}
}
function handleWork($farmId)
{
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) {
for ($i = 0; $i < 10; $i++) {
if ($DBFarmRole->NewRoleID != '') {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled.");
continue 2;
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) == '0' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping...");
continue 2;
}
// Get polling interval in seconds
$polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60;
$dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME);
if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) {
$this->logger->info("Polling interval: every {$polling_interval} seconds");
//continue;
}
// Set Last polling time
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time());
// Get current count of running and pending instances.
$this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue 2;
} elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instance�s count after scaling
resolution the spare instances are running�g for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
// Select instance that will be terminated
//
// * Instances ordered by uptime (oldest wil be choosen)
// * Instance cannot be mysql master
// * Choose the one that was rebundled recently
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED)));
if ($DBServer->index == 1 && $serversCount > 1) {
continue;
}
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
/*
* We do not want to delete the most recently synced instance. Because of LA fluctuation.
* I.e. LA may skyrocket during sync and drop dramatically after sync.
*/
if ($DBServer->dateLastSync != 0) {
$chk_sync_time = $this->db->GetOne("SELECT server_id FROM servers \n\t \t\tWHERE dtlastsync > {$DBServer->dateLastSync} \n\t\t \tAND farm_roleid='{$DBServer->farmRoleId}' AND status != '" . SERVER_STATUS::TERMINATED . "'");
if ($chk_sync_time) {
$got_valid_instance = true;
}
} else {
$got_valid_instance = true;
}
}
}
if ($DBServer && $got_valid_instance) {
$this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$AmazonEC2Client = Scalr_Service_Cloud_Aws::newEc2($DBServer->GetProperty(EC2_SERVER_PROPERTIES::REGION), $DBServer->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), $DBServer->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE));
// Shutdown an instance just before a full hour running
$response = $AmazonEC2Client->DescribeInstances($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID));
if ($response && $response->reservationSet->item) {
$launch_time = strtotime($response->reservationSet->item->instancesSet->item->launchTime);
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->reservationSet->item->instancesSet->item->launchTime)));
}
}
//
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
//Check safe shutdown
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) {
$snmpClient = new Scalr_Net_Snmp_Client();
$port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT);
$snmpClient->connect($DBServer->remoteIp, $port ? $port : 161, $DBFarm->Hash, null, null, false);
$res = $snmpClient->get('1.3.6.1.4.1.36632.6.1');
if ($res != '1') {
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script return '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res)));
}
}
try {
Scalr::FireEvent($DBFarm->ID, new BeforeHostTerminateEvent($DBServer, false));
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time());
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
$this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
}
} else {
$this->logger->warn(sprintf("[FarmID: {$DBFarm->ID}] Scalr unable to determine what instance it should terminate (FarmRoleID: {$DBFarmRole->ID}). Skipping..."));
}
break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution �need more instances� for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count increase
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name)));
continue 2;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=?", array($DBFarm->ID));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time());
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//It must be done for any worker in daemon mode. (Policy engine, PM, etc.. settings shouldn't be cached)
\Scalr::getContainer()->warmup();
try {
$dbServer = DBServer::LoadByID($request->serverId);
if ($dbServer->status == SERVER_STATUS::TEMPORARY) {
try {
$dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
} catch (Exception $e) {
}
} else {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$dbServer->Remove();
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_LAUNCH) {
$account = Scalr_Account::init()->loadById($dbServer->clientId);
if ($account->status == Scalr_Account::STATUS_ACTIVE) {
\Scalr::LaunchServer(null, $dbServer);
}
}
}
}
} catch (Exception $e) {
$this->getLogger()->error("Server: %s, manager failed with exception: %s", $request->serverId, $e->getMessage());
}
return $request;
}
/**
* Farm launched
*
* @param FarmLaunchedEvent $event
*/
public function OnFarmLaunched(FarmLaunchedEvent $event)
{
$DBFarm = DBFarm::LoadByID($this->FarmID);
// TODO: Refactoting -> Move to DBFarm class
$this->DB->Execute("UPDATE farms SET status=?, dtlaunched=NOW() WHERE id=?", array(FARM_STATUS::RUNNING, $this->FarmID));
$governance = new Scalr_Governance($DBFarm->EnvID);
if ($governance->isEnabled(Scalr_Governance::GENERAL_LEASE) && $DBFarm->GetSetting(DBFarm::SETTING_LEASE_STATUS)) {
$dt = new DateTime();
$dt->add(new DateInterval('P' . intval($governance->getValue(Scalr_Governance::GENERAL_LEASE, 'defaultLifePeriod')) . 'D'));
$DBFarm->SetSetting(DBFarm::SETTING_LEASE_EXTEND_CNT, 0);
$DBFarm->SetSetting(DBFarm::SETTING_LEASE_TERMINATE_DATE, $dt->format('Y-m-d H:i:s'));
$DBFarm->SetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND, '');
}
$roles = $DBFarm->GetFarmRoles();
foreach ($roles as $dbFarmRole) {
if ($dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) && !$DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
$ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, true, "Farm launched", isset($event->userId) ? $event->userId : null);
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $dbFarmRole->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
function handleWork($farmId)
{
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) {
for ($i = 0; $i < 10; $i++) {
if ($DBFarmRole->NewRoleID != '') {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled.");
continue 2;
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) != '1' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
$this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping...");
continue 2;
}
// Get polling interval in seconds
$polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60;
$dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME);
if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) {
$this->logger->info("Polling interval: every {$polling_interval} seconds");
continue;
}
// Set Last polling time
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time(), DBFarmRole::TYPE_LCL);
// Get current count of running and pending instances.
$this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue 2;
} elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instance�s count after scaling
resolution the spare instances are running�g for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
// Select instance that will be terminated
//
// * Instances ordered by uptime (oldest wil be choosen)
// * Instance cannot be mysql master
// * Choose the one that was rebundled recently
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))));
if ($DBServer->index == 1 && $serversCount > 1) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
// We do not want to delete the most recently synced instance. Because of LA fluctuation.
// I.e. LA may skyrocket during sync and drop dramatically after sync.
if ($DBServer->dateLastSync != 0) {
$chk_sync_time = $this->db->GetOne("\n SELECT server_id FROM servers\n WHERE dtlastsync > {$DBServer->dateLastSync}\n AND farm_roleid='{$DBServer->farmRoleId}'\n AND status NOT IN('" . SERVER_STATUS::TERMINATED . "', '" . SERVER_STATUS::TROUBLESHOOTING . "')\n LIMIT 1\n ");
if ($chk_sync_time) {
$got_valid_instance = true;
}
} else {
$got_valid_instance = true;
}
}
}
if ($DBServer && $got_valid_instance) {
$this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_IGNORE_FULL_HOUR)) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c'))));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
//Check safe shutdown
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) {
if ($DBServer->IsSupported('0.11.3')) {
try {
$port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$port) {
$port = 8010;
}
$szrClient = Scalr_Net_Scalarizr_Client::getClient($DBServer, Scalr_Net_Scalarizr_Client::NAMESPACE_SYSTEM, $port);
$res = $szrClient->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled, but not supported by scalarizr installed on server '%s'. Ignoring.", $DBServer->serverId)));
}
if ($res != '1') {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res)));
break;
}
}
try {
$DBServer->terminate('SCALING_DOWN', false);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
$this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
}
} else {
$this->logger->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $DBFarm->ID, $DBFarmRole->ID));
}
break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution �need more instances� for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue 2;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name)));
continue 2;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($DBFarm->ID));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Scaling up");
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
/**
* Farm launched
*
* @param FarmLaunchedEvent $event
*/
public function OnFarmLaunched(FarmLaunchedEvent $event)
{
$DBFarm = DBFarm::LoadByID($this->FarmID);
// TODO: Refactoting -> Move to DBFarm class
$this->DB->Execute("UPDATE farms SET status=?, dtlaunched=NOW() WHERE id=?", array(FARM_STATUS::RUNNING, $this->FarmID));
$roles = $DBFarm->GetFarmRoles();
foreach ($roles as $dbFarmRole) {
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
$ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, true);
$dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time());
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $dbFarmRole->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
public function StartThread($bundle_task_info)
{
$db = \Scalr::getDb();
// Reconfigure observers;
Scalr::ReconfigureObservers();
$BundleTask = BundleTask::LoadById($bundle_task_info['id']);
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
} catch (\Scalr\Exception\ServerNotFoundException $e) {
if (!$BundleTask->snapshotId) {
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$BundleTask->setDate('finished');
$BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId);
$BundleTask->Save();
return;
}
} catch (Exception $e) {
//$this->Logger->error($e->getMessage());
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION:
$conn = @fsockopen($DBServer->getSzrHost(), $DBServer->getPort(DBServer::PORT_CTRL), $errno, $errstr, 10);
if ($conn) {
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1);
$BundleTask->Log("Outbound connection successfully established. Awaiting user action: prebuild automation selection");
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION;
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
$BundleTask->Save();
} else {
$errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $requestHost, $ctrlPort, $errstr);
$errMsg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR);
if (!$errMsg || $errstr != $errMsg) {
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr);
$BundleTask->Log("{$errstr} Will try again in a few minutes.");
}
}
exit;
break;
case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION:
//NOTHING TO DO;
exit;
break;
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$BundleTask->setDate('started');
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($DBServer->platform)->GetServerID($DBServer)) {
$BundleTask->Log(sprintf(_("Waiting for temporary server")));
exit;
}
if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
// IF server is in pensing state
$status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer);
if ($status->isPending()) {
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
$BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName()));
exit;
} elseif ($status->isTerminated()) {
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
break;
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$DBServer->remoteIp = $ips['remoteIp'];
$DBServer->localIp = $ips['localIp'];
$DBServer->save();
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$BundleTask->save();
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$BundleTask->Log(sprintf(_("Initializing SSH2 session to the server")));
if ($DBServer->platform == SERVER_PLATFORMS::IDCF && !$DBServer->remoteIp) {
try {
$BundleTask->Log("Creating port forwarding rules to be able to connect to the server by SSH");
$environment = $DBServer->GetEnvironmentObject();
$cloudLocation = $DBServer->GetCloudLocation();
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$sharedIpId = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP_ID . ".{$cloudLocation}", $environment, false);
$sharedIp = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP . ".{$cloudLocation}", $environment, false);
$BundleTask->Log("Shared IP: {$sharedIp}");
$cs = $environment->cloudstack($DBServer->platform);
// Create port forwarding rules for scalarizr
$port = $platform->getConfigVariable(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false);
if (!$port) {
$port1 = 30000;
$port2 = 30001;
$port3 = 30002;
$port4 = 30003;
} else {
$port1 = $port + 1;
$port2 = $port1 + 1;
$port3 = $port2 + 1;
$port4 = $port3 + 1;
}
$virtualmachineid = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID);
$result2 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8014, 'protocol' => "udp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid));
$result1 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8013, 'protocol' => "tcp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid));
$result3 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8010, 'protocol' => "tcp", 'publicport' => $port3, 'virtualmachineid' => $virtualmachineid));
$result4 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8008, 'protocol' => "tcp", 'publicport' => $port2, 'virtualmachineid' => $virtualmachineid));
$result5 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 22, 'protocol' => "tcp", 'publicport' => $port4, 'virtualmachineid' => $virtualmachineid));
$DBServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4));
$DBServer->remoteIp = $sharedIp;
$DBServer->Save();
$platform->setConfigVariable(array(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false);
} catch (Exception $e) {
$BundleTask->Log("Unable to create port-forwarding rules: {$e->getMessage()}");
}
exit;
}
if ($DBServer->platform == SERVER_PLATFORMS::ECS && !$DBServer->remoteIp) {
$BundleTask->Log(sprintf(_("Server doesn't have public IP. Assigning...")));
$osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION));
$ports = $osClient->network->ports->list();
foreach ($ports as $port) {
if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$serverNetworkPort = $port->id;
break;
}
}
$ips = $osClient->network->floatingIps->list();
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ipId = false;
$ipInfo = false;
foreach ($ips as $ip) {
if ($ip->port_id && $ip->port_id == $serverNetworkPort) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipAssigned = true;
$ipInfo = $ip;
break;
}
if (!$ip->fixed_ip_address && !$ipAddress) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipInfo = $ip;
}
}
if (!$ipAssigned) {
if (!$serverNetworkPort) {
$BundleTask->Log("Unable to identify network port of instance");
exit;
} else {
if (!$ipAddress) {
$networks = $osClient->network->listNetworks();
foreach ($networks as $network) {
if ($network->{"router:external"} == true) {
$publicNetworkId = $network->id;
}
}
if (!$publicNetworkId) {
$BundleTask->Log("Unable to identify public network to allocate");
exit;
} else {
$ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort);
$ipAddress = $ip->floating_ip_address;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
$BundleTask->Log("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}");
}
} else {
$BundleTask->Log("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")");
$osClient->network->floatingIps->update($ipId, $serverNetworkPort);
}
}
} else {
$BundleTask->Log("IP: {$ipAddress} already assigned");
}
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
exit;
}
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log(sprintf(_("Created SSH session. Username: %s"), $ssh2Client->getLogin()));
//Prepare script
$BundleTask->Log(sprintf(_("Uploading builder scripts...")));
$behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
if ($DBServer->isOpenstack()) {
$platform = SERVER_PLATFORMS::OPENSTACK;
} else {
$platform = $DBServer->platform;
}
$baseUrl = rtrim(\Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host'), '/');
$options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $DBServer->envId, 'region' => $DBServer->GetCloudLocation(), 'scalr-id' => SCALR_ID);
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
/*
%CHEF_SERVER_URL%
%CHEF_VALIDATOR_NAME%
%CHEF_VALIDATOR_KEY%
%CHEF_ENVIRONMENT%
%CHEF_ROLE_NAME%
*/
$chefServerId = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID);
if ($chefServerId) {
$chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId));
$chefServerInfo['v_auth_key'] = $this->crypto->decrypt($chefServerInfo['v_auth_key'], $this->cryptoKey);
}
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
/*
$BundleTask->Log(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
*/
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
// For CGE we need to use sudo
if ($BundleTask->platform == SERVER_PLATFORMS::GCE || $BundleTask->osFamily == 'amazon') {
$shell = $ssh2Client->getShell();
@stream_set_blocking($shell, true);
@stream_set_timeout($shell, 5);
@fwrite($shell, "sudo touch /var/log/role-builder-output.log 2>&1" . PHP_EOL);
$output = @fgets($shell, 4096);
$BundleTask->Log("Verbose 1: {$output}");
@fwrite($shell, "sudo chmod 0666 /var/log/role-builder-output.log 2>&1" . PHP_EOL);
$output2 = @fgets($shell, 4096);
$BundleTask->Log("Verbose 2: {$output2}");
@fwrite($shell, "sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &" . PHP_EOL);
$output3 = @fgets($shell, 4096);
$BundleTask->Log("Verbose 3: {$output3}");
sleep(5);
$meta = stream_get_meta_data($shell);
$BundleTask->Log(sprintf("Verbose (Meta): %s", json_encode($meta)));
$i = 4;
if ($meta['eof'] == false && $meta['unread_bytes'] != 0) {
$output4 = @fread($shell, $meta['unread_bytes']);
$BundleTask->Log("Verbose {$i}: {$output4}");
$meta = stream_get_meta_data($shell);
$BundleTask->Log(sprintf("Verbose (Meta): %s", json_encode($meta)));
}
@fclose($shell);
/*
$r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log");
$BundleTask->Log("1: {$r1} ({$ssh2Client->stdErr})");
$r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log");
$BundleTask->Log("2: {$r2} ({$ssh2Client->stdErr})");
$r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$BundleTask->Log("3: {$r3} ({$ssh2Client->stdErr})");
*/
} else {
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$BundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog));
$BundleTask->SnapshotCreationFailed($msg);
} else {
$BundleTask->Log($msg);
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($BundleTask->platform);
$platformModule->CreateServerSnapshot($BundleTask);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage());
$BundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($BundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS:
$r_farm_roles = array();
$BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType));
try {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$BundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=? AND new_role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
$update_farm_dns_zones = array();
$completed_roles = 0;
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) {
$BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
$completed_roles++;
} else {
$servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. " . "Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
if (count($servers) == 0) {
$DBFarmRole->RoleID = $DBFarmRole->NewRoleID;
$DBFarmRole->NewRoleID = null;
$DBFarmRole->Save();
$update_farm_dns_zones[$DBFarmRole->FarmID] = 1;
$completed_roles++;
} else {
$metaData = $BundleTask->getSnapshotDetails();
foreach ($servers as $server) {
try {
$DBServer = DBServer::LoadByID($server['server_id']);
} catch (Exception $e) {
//TODO:
continue;
}
if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) {
$DBServer->roleId = $BundleTask->roleId;
$DBServer->Save();
if ($metaData['noServersReplace']) {
$BundleTask->Log(sprintf("'Do not replace servers' option was checked. " . "Server '%s' won't be replaced to new image.", $DBServer->serverId));
} else {
$BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, " . "already has all modifications. No need to replace it.", $DBServer->serverId));
}
} else {
if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?,?) LIMIT 1", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TROUBLESHOOTING, SERVER_STATUS::SUSPENDED))) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID);
$nDBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_REPLACE_SERVER_FROM_SNAPSHOT, !empty($BundleTask->createdById) ? $BundleTask->createdById : null);
$nDBServer->replaceServerID = $DBServer->serverId;
$nDBServer->Save();
$BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId));
}
}
}
}
}
}
if ($completed_roles == count($r_farm_roles)) {
$BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
$DBServer->Remove();
} elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) {
$BundleTask->Log("Terminating temporary server");
$DBServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
$BundleTask->Log("Termination request has been sent");
}
} catch (Exception $e) {
$BundleTask->Log("Warning: {$e->getMessage()}");
}
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Save();
$BundleTask->createImageEntity();
}
try {
if (count($update_farm_dns_zones) != 0) {
foreach ($update_farm_dns_zones as $farm_id => $v) {
$dnsZones = DBDNSZone::loadByFarmId($farm_id);
foreach ($dnsZones as $dnsZone) {
if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) {
$dnsZone->updateSystemRecords();
$dnsZone->save();
}
}
}
}
} catch (Exception $e) {
$this->Logger->fatal("DNS ZONE: {$e->getMessage()}");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = DBRole::loadById($DBServer->roleId);
if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($DBServer) {
$new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer);
} else {
$new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
} else {
//TODO:
//$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})");
}
}
if ($BundleTask->object == BundleTask::BUNDLETASK_OBJECT_ROLE) {
try {
$DBRole = DBRole::createFromBundleTask($BundleTask);
} catch (Exception $e) {
$BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
} else {
// roleName is empty, just create image
$BundleTask->createImageEntity();
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
if ($DBServer->farmId) {
// Create DBFarmRole object
// TODO: create DBFarm role
}
}
} catch (Exception $e) {
}
} else {
try {
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS));
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
$DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
} else {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
$DBFarmRole = DBFarmRole::LoadByID($farm_role['id']);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
}
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS;
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
$BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
if ($BundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) {
try {
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
$DBServer->Remove();
} elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) {
$BundleTask->Log("Terminating temporary server");
$DBServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
$BundleTask->Log("Termination request has been sent");
}
} catch (Exception $e) {
$BundleTask->Log("Warning: {$e->getMessage()}");
}
}
$BundleTask->Save();
$BundleTask->createImageEntity();
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
}
break;
}
}
public function ServerLaunch($FarmRoleID, $IncreaseMaxInstances = false)
{
try {
$DBFarmRole = DBFarmRole::LoadByID($FarmRoleID);
$DBFarm = DBFarm::LoadByID($DBFarmRole->FarmID);
} catch (Exception $e) {
throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID));
}
if ($DBFarm->EnvID != $this->Environment->id) {
throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID));
}
//TODO: Remove this limitation
/*
$isSzr = $dbFarmRole->GetRoleObject()->isSupported("0.5");
if ($retval == Scalr_Scaling_Decision::UPSCALE && ($dbFarmRole->GetPendingInstancesCount() > 5 || !$isSzr))
*/
$isSzr = $DBFarmRole->GetRoleObject()->isSupported("0.5");
$n = $DBFarmRole->GetPendingInstancesCount();
if ($n >= 5 && !$isSzr) {
throw new Exception("There are {$n} pending instances. You cannot launch new instances while you have 5 pending ones.");
}
$response = $this->CreateInitialResponse();
$max_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES);
$min_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES);
if ($IncreaseMaxInstances) {
if ($max_instances < $min_instances + 1) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1);
}
}
if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() >= $max_instances) {
if ($IncreaseMaxInstances) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1);
} else {
throw new Exception("Max instances limit reached. Use 'IncreaseMaxInstances' parameter or increase max isntances settings in UI");
}
}
if ($DBFarmRole->GetRunningInstancesCount() < $min_instances || $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES) < $min_instances) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $min_instances + 1);
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Starting new instance (API). ServerID = %s.", $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::API)->error($e->getMessage());
}
$response->ServerID = $DBServer->serverId;
return $response;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
try {
$dbServer = DBServer::LoadByID($request->serverId);
if ($dbServer->status == SERVER_STATUS::TEMPORARY) {
try {
$dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
} catch (Exception $e) {
}
} else {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$dbServer->Remove();
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_LAUNCH) {
$account = Scalr_Account::init()->loadById($dbServer->clientId);
if ($account->status == Scalr_Account::STATUS_ACTIVE) {
\Scalr::LaunchServer(null, $dbServer);
}
}
}
}
} catch (Exception $e) {
$this->getLogger()->error("Server: %s, manager failed with exception: %s", $request->serverId, $e->getMessage());
}
return $request;
}
private function cleanup()
{
// Check that time has come to cleanup dead servers
$doCleanup = false;
sem_acquire($this->cleanupSem);
try {
if (time() - (int) $this->lastCleanup->get(0) >= $this->cleanupInterval) {
$doCleanup = true;
$this->lastCleanup->put(0, time());
}
} catch (Exception $e) {
sem_release($this->cleanupSem);
}
sem_release($this->cleanupSem);
if ($doCleanup) {
$this->logger->info("Cleanup dead servers");
try {
$terminated_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=? AND (UNIX_TIMESTAMP(dtshutdownscheduled)+3600 < UNIX_TIMESTAMP(NOW()) OR dtshutdownscheduled IS NULL)", array(SERVER_STATUS::TERMINATED));
foreach ($terminated_servers as $ts) {
DBServer::LoadByID($ts['server_id'])->Remove();
}
$p_terminated_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=? AND role_id='0' AND farm_id IS NULL", array(SERVER_STATUS::PENDING_TERMINATE));
foreach ($p_terminated_servers as $ts) {
DBServer::LoadByID($ts['server_id'])->Remove();
}
$importing_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status IN(?,?) AND UNIX_TIMESTAMP(dtadded)+86400 < UNIX_TIMESTAMP(NOW())", array(SERVER_STATUS::IMPORTING, SERVER_STATUS::TEMPORARY));
foreach ($importing_servers as $ts) {
DBServer::LoadByID($ts['server_id'])->Remove();
}
$pending_launch_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=?", array(SERVER_STATUS::PENDING_LAUNCH));
try {
foreach ($pending_launch_servers as $ts) {
$DBServer = DBServer::LoadByID($ts['server_id']);
$account = Scalr_Account::init()->loadById($DBServer->clientId);
if ($account->status == Scalr_Account::STATUS_ACTIVE) {
Scalr::LaunchServer(null, $DBServer);
}
}
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Can't load server with ID #'%s'", $ts['server_id'], $e->getMessage()));
}
} catch (Exception $e) {
$this->logger->fatal("Poller::cleanup failed: {$e->getMessage()}");
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (!isset($request->farmRoleId)) {
//This is the farm with synchronous launch of roles
try {
$DBFarm = DBFarm::LoadByID($request->farmId);
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID);
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId);
throw $e;
}
//Gets the list of the roles
$list = $DBFarm->GetFarmRoles();
} else {
//This is asynchronous lauhch
try {
$DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId);
if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) {
//We don't need to handle inactive farms
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId);
throw $e;
}
$list = [$DBFarmRole];
}
$this->getLogger()->debug("Processing %s FarmRoles", count($list));
foreach ($list as $DBFarmRole) {
// Set Last polling time
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$disabledScaling = false;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') {
if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
// For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running)
// When 1 instance is already running, the rest is fully manual
$roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount();
if ($roleTotalInstances != 0) {
$disabledScaling = true;
}
} else {
if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$disabledScaling = true;
}
}
if ($disabledScaling) {
$this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias);
continue;
}
}
$farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name;
// Get current count of running and pending instances.
$this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
//Replacing the logger
$scalingManager->logger = $this->getLogger();
$scalingDecision = $scalingManager->makeScalingDecision();
$scalingDecisionDetails = $scalingManager->decisonInfo;
$this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails));
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue;
} else {
if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instances count after scaling
resolution the spare instances are running for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null));
continue;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
$ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR);
$useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN);
$isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ);
// Select instance that will be terminated
//
// Instances ordered by uptime (oldest wil be choosen)
// Instance cannot be mysql master
// Choose the one that was rebundled recently
$DBServer = null;
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($isRabbitMQ) {
$serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]);
if ($DBServer->index == 1 && $serverExists) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
$got_valid_instance = true;
}
//Check safe shutdown
if ($useSafeShutdown == 1) {
try {
$res = $DBServer->scalarizr->system->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
if ($res != '1') {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
$got_valid_instance = false;
}
}
}
// end while
if ($DBServer !== null && $got_valid_instance) {
$this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$ignoreFullHour) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
try {
if ($terminateStrategy == 'terminate') {
$DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} else {
$DBServer->suspend('SCALING_DOWN', false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
} catch (Exception $e) {
$this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId));
}
}
} else {
$this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID));
}
//break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution 'need more instances' for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias));
continue;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName)));
continue;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it.");
return;
}
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
$suspendedServer = null;
if ($terminateStrategy == 'suspend') {
$suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED));
if (count($suspendedServers) > 0) {
$suspendedServer = array_shift($suspendedServers);
}
}
if ($terminateStrategy == 'suspend' && $suspendedServer) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId));
}
if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
} else {
$platform = PlatformFactory::NewPlatform($suspendedServer->platform);
$platform->ResumeServer($suspendedServer);
}
}
}
}
return true;
}
/**
* {@inheritdoc}
* @see \Scalr\Observer\AbstractEventObserver::OnFarmLaunched()
*/
public function OnFarmLaunched(FarmLaunchedEvent $event)
{
$DBFarm = DBFarm::LoadByID($this->FarmID);
$this->DB->Execute("UPDATE farms SET status = ?, dtlaunched = NOW() WHERE id = ? LIMIT 1", [FARM_STATUS::RUNNING, $this->FarmID]);
$this->getContainer()->auditlogger->log('farm.launch', $DBFarm, $event->auditLogExtra);
$governance = new Scalr_Governance($DBFarm->EnvID);
if ($governance->isEnabled(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE) && $DBFarm->GetSetting(Entity\FarmSetting::LEASE_STATUS)) {
$dt = new DateTime();
$dt->add(new DateInterval('P' . intval($governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'defaultLifePeriod')) . 'D'));
$DBFarm->SetSetting(Entity\FarmSetting::LEASE_EXTEND_CNT, 0);
$DBFarm->SetSetting(Entity\FarmSetting::LEASE_TERMINATE_DATE, $dt->format('Y-m-d H:i:s'));
$DBFarm->SetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND, '');
}
$roles = $DBFarm->GetFarmRoles();
foreach ($roles as $dbFarmRole) {
if ($dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) && !$DBFarm->GetSetting(Entity\FarmSetting::EC2_VPC_ID)) {
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingDecision = $scalingManager->makeScalingDecision();
if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
$ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, true, DBServer::LAUNCH_REASON_FARM_LAUNCHED, isset($event->userId) ? $event->userId : null);
$dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$role = $dbFarmRole->GetRoleObject();
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", !empty($DBFarm->Name) ? $DBFarm->Name : null, !empty($role->name) ? $role->name : null, !empty($DBServer->serverId) ? $DBServer->serverId : null), !empty($DBServer->serverId) ? $DBServer->serverId : null, !empty($DBServer->envId) ? $DBServer->envId : null, !empty($DBServer->farmRoleId) ? $DBServer->farmRoleId : null));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
