Launches server
public static LaunchServer ( ServerCreateInfo $ServerCreateInfo = null, DBServer $DBServer = null, boolean $delayed = false, integer | array $reason, Scalr_Account_User | integer $user = null ) : DBServer | null | ||
$ServerCreateInfo | ServerCreateInfo | optional The server create info |
$DBServer | DBServer | optional The DBServer object |
$delayed | boolean | optional |
$reason | integer | array | optional |
$user | Scalr_Account_User | integer | optional The Scalr_Account_User object or its unique identifier |
return | DBServer | null | Returns the DBServer object on cussess or null otherwise |
/** * {@inheritdoc} * @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::startForking() */ function startForking($workQueue) { // Reopen DB connection after daemonizing $this->db = $this->getContainer()->adodb; // Check that time has come to cleanup dead servers $doCleanup = false; sem_acquire($this->cleanupSem); try { if (time() - (int) $this->lastCleanup->get(0) >= $this->cleanupInterval) { $doCleanup = true; $this->lastCleanup->put(0, time()); } } catch (Exception $e) { sem_release($this->cleanupSem); } sem_release($this->cleanupSem); if ($doCleanup) { $this->logger->info("Cleanup dead servers"); try { $importing_servers = $this->db->GetAll("\n SELECT server_id FROM servers\n WHERE status IN(?, ?)\n AND `dtadded` < NOW() - INTERVAL 1 DAY\n ", array(SERVER_STATUS::IMPORTING, SERVER_STATUS::TEMPORARY)); foreach ($importing_servers as $ts) { $dbServer = DBServer::LoadByID($ts['server_id']); if ($dbServer->status == SERVER_STATUS::TEMPORARY) { try { $dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); } catch (Exception $e) { } } else { if ($dbServer->status == SERVER_STATUS::IMPORTING) { $dbServer->Remove(); } } } $pending_launch_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=?", array(SERVER_STATUS::PENDING_LAUNCH)); try { foreach ($pending_launch_servers as $ts) { $DBServer = DBServer::LoadByID($ts['server_id']); if ($DBServer->status == SERVER_STATUS::PENDING_LAUNCH) { $account = Scalr_Account::init()->loadById($DBServer->clientId); if ($account->status == Scalr_Account::STATUS_ACTIVE) { Scalr::LaunchServer(null, $DBServer); } } } } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Can't load server with ID #'%s'", $ts['server_id'], $e->getMessage())); } } catch (Exception $e) { $this->logger->fatal("Poller::cleanup failed: {$e->getMessage()}"); } } }
public function xLaunchNewServerAction() { $dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId')); $dbFarm = $dbFarmRole->GetFarmObject(); $this->user->getPermissions()->validate($dbFarmRole); if ($dbFarm->Status != FARM_STATUS::RUNNING) { throw new Exception("You can launch servers only on running farms"); } $dbRole = $dbFarmRole->GetRoleObject(); if ($dbRole->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { throw new Exception("Manual launch of VPC Router insatnces is not allowed"); } $pendingInstancesCount = $dbFarmRole->GetPendingInstancesCount(); $maxInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES); $minInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES); if ($maxInstances < $minInstances + 1) { $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $maxInstances + 1, DBFarmRole::TYPE_CFG); $warnMsg = sprintf(_("Server count has been increased. Scalr will now request a new server from your cloud. Since the server count was already at the maximum set for this role, we increased the maximum by one."), $dbRole->name, $dbRole->name); } $runningInstancesCount = $dbFarmRole->GetRunningInstancesCount(); if ($runningInstancesCount + $pendingInstancesCount >= $minInstances) { $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $minInstances + 1, DBFarmRole::TYPE_CFG); } $serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); Scalr::LaunchServer($serverCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY, $this->user); if ($warnMsg) { $this->response->warning($warnMsg); } else { $this->response->success('Server successfully launched'); } }
/** * Launches new server * @param int $farmRoleId * @param bool $increaseMinInstances * @param bool $needConfirmation * @throws Scalr_Exception_Core */ public function xLaunchNewServerAction($farmRoleId, $increaseMinInstances = false, $needConfirmation = true) { $dbFarmRole = DBFarmRole::LoadByID($farmRoleId); $dbFarm = $dbFarmRole->GetFarmObject(); $this->user->getPermissions()->validate($dbFarmRole); $this->request->restrictFarmAccess($dbFarm, Acl::PERM_FARMS_SERVERS); if ($dbFarm->Status != FARM_STATUS::RUNNING) { throw new Scalr_Exception_Core('You can launch servers only on running farms'); } $dbRole = $dbFarmRole->GetRoleObject(); if ($dbRole->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { throw new Scalr_Exception_Core('Manual launch of VPC Router insatnces is not allowed'); } if ($dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) == 1) { $scalingManager = new Scalr_Scaling_Manager($dbFarmRole); $scalingMetrics = $scalingManager->getFarmRoleMetrics(); $hasScalingMetrics = count($scalingMetrics) > 0; $curInstances = $dbFarmRole->GetPendingInstancesCount() + $dbFarmRole->GetRunningInstancesCount(); $maxInstances = $dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_MAX_INSTANCES); $minInstances = $dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_MIN_INSTANCES); if ($needConfirmation) { $res = ['showConfirmation' => true]; if ($maxInstances < $curInstances + 1) { $res['showIncreaseMaxInstancesWarning'] = true; $res['maxInstances'] = $maxInstances + 1; } if ($hasScalingMetrics && $curInstances >= $minInstances) { $res['showIncreaseMinInstancesConfirm'] = true; } $this->response->data($res); return; } else { if ($maxInstances < $curInstances + 1) { $dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_MAX_INSTANCES, $maxInstances + 1, Entity\FarmRoleSetting::TYPE_CFG); } if ($increaseMinInstances && $hasScalingMetrics && $curInstances >= $minInstances) { $dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_MIN_INSTANCES, $minInstances + 1, Entity\FarmRoleSetting::TYPE_CFG); } } } $serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); Scalr::LaunchServer($serverCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY, $this->user); $this->response->success('Server successfully launched'); }
public function StartThread($bundle_task_info) { $db = Core::GetDBInstance(); // Reconfigure observers; Scalr::ReconfigureObservers(); $BundleTask = BundleTask::LoadById($bundle_task_info['id']); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); } catch (ServerNotFoundException $e) { if (!$BundleTask->snapshotId) { $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED; $BundleTask->setDate('finished'); $BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId); $BundleTask->Save(); return; } } catch (Exception $e) { //$this->Logger->error($e->getMessage()); } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } // IF server is in pensing state $status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer); $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); if ($status->isPending()) { $BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName())); exit; } elseif ($status->isTerminated()) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } break; } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $DBServer->remoteIp = $ips['remoteIp']; $DBServer->localIp = $ips['locateIp']; $DBServer->save(); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV; $BundleTask->save(); $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: $BundleTask->Log(sprintf(_("Initializing SSH2 session to the server"))); try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, 22); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage())); //TODO: Set status of bundle log to failed exit; } //Prepare script $BundleTask->Log(sprintf(_("Uploading builder scripts..."))); $behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR); try { $options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $DBServer->platform, 'behaviour' => trim(str_replace("base", "", $behaviors)), 'queryenv-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/query-env", 'messaging-p2p.producer-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/messaging"); $command = 'scalarizr --import -y'; foreach ($options as $k => $v) { $command .= sprintf(' -o %s=%s', $k, $v); } if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') { $recipes = 'mysql=percona'; } else { $recipes = ''; } $scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH); $scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl"); $scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_NODE_NAME%", "\r\n"), array($DBServer->platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', '', '', '', '', '', '', "\n"), $scriptContents); if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) { throw new Exception("Cannot upload script"); } $BundleTask->Log(sprintf(_("Uploading chef recipes..."))); if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) { throw new Exception("Cannot upload chef recipes"); } } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $BundleTask->Log("Launching role builder routines on server"); $ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh"); $ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE; $BundleTask->save(); break; case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, 22); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $log = $ssh2Client->getFile('/var/log/role-builder-output.log'); $log_lines = explode("\r\n", $log); $last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE); while ($msg = trim(array_shift($log_lines))) { if (substr($msg, -1, 1) != ']') { continue; } if ($last_msg) { if ($msg != $last_msg) { continue; } elseif ($msg == $last_msg) { $last_msg = null; continue; } } if (stristr($msg, '[ Failed ]')) { $stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log'); $BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog)); $BundleTask->SnapshotCreationFailed($msg); /** Terminate server **/ PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $DBServer->status = SERVER_STATUS::PENDING_TERMINATE; $DBServer->save(); $BundleTask->Log(sprintf("Temporary server '%s' (%s) has been terminated", $DBServer->serverId, $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))); } else { $BundleTask->Log($msg); $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg); } } //Read /var/log/role-builder-output.log break; case SERVER_SNAPSHOT_CREATION_STATUS::PENDING: try { $platformModule = PlatformFactory::NewPlatform($BundleTask->platform); $platformModule->CreateServerSnapshot($BundleTask); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage()); $BundleTask->SnapshotCreationFailed($e->getMessage()); } break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING: $addedTime = strtotime($BundleTask->dateAdded); if ($addedTime + 3600 < time()) { $BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time."); } break; case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS: PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask); break; case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS: $r_farm_roles = array(); $BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType)); try { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); } catch (Exception $e) { if (stristr($e->getMessage(), "not found in database")) { $BundleTask->SnapshotCreationFailed("Farm was removed before task was finished"); } return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { try { $r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); } catch (Exception $e) { } } elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND new_role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { try { $r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']); } catch (Exception $e) { } } } $update_farm_dns_zones = array(); $completed_roles = 0; foreach ($r_farm_roles as $DBFarmRole) { if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) { $BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); $completed_roles++; } else { $servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)); $BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); if (count($servers) == 0) { $DBFarmRole->RoleID = $DBFarmRole->NewRoleID; $DBFarmRole->NewRoleID = null; $DBFarmRole->Save(); $update_farm_dns_zones[$DBFarmRole->FarmID] = 1; $completed_roles++; } else { $metaData = $BundleTask->getSnapshotDetails(); foreach ($servers as $server) { try { $DBServer = DBServer::LoadByID($server['server_id']); } catch (Exception $e) { //TODO: continue; } if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) { $DBServer->roleId = $BundleTask->roleId; $DBServer->Save(); if ($metaData['noServersReplace']) { $BundleTask->Log(sprintf("'Do not replace servers' option was checked. Server '%s' won't be replaced to new image.", $DBServer->serverId)); } else { $BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, already has all modifications. No need to replace it.", $DBServer->serverId)); } if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) { PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t\t\t\t\t\t", array(sprintf("Farm was in 'Synchronizing' state. Server terminated when bundling was completed. Bundle task #%s", $BundleTask->id), $DBServer->serverId)); } } else { if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?)", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE))) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID); $nDBServer = Scalr::LaunchServer($ServerCreateInfo); $nDBServer->replaceServerID = $DBServer->serverId; $nDBServer->Save(); $BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId)); } } // if serverid != bundletask->serverID } // foreach server } // count($servers) } } if ($completed_roles == count($r_farm_roles)) { $BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Save(); } try { if (count($update_farm_dns_zones) != 0) { foreach ($update_farm_dns_zones as $farm_id => $v) { $dnsZones = DBDNSZone::loadByFarmId($farm_id); foreach ($dnsZones as $dnsZone) { if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) { $dnsZone->updateSystemRecords(); $dnsZone->save(); } } } } } catch (Exception $e) { $this->Logger->fatal("DNS ZONE: {$e->getMessage()}"); } break; case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE: try { if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $saveOldRole = false; try { $dbRole = DBRole::loadById($DBServer->roleId); if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) { $saveOldRole = true; } } catch (Exception $e) { //NO OLD ROLE } if ($dbRole && $saveOldRole) { if ($DBServer) { $new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer); } else { $new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999); } $dbRole->name = $new_role_name; $BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name)); $dbRole->save(); } else { //TODO: //$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})"); } } try { $DBRole = DBRole::createFromBundleTask($BundleTask); } catch (Exception $e) { $BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again."); return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) { $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); if ($DBServer->status == SERVER_STATUS::IMPORTING) { if ($DBServer->farmId) { // Create DBFarmRole object // TODO: create DBFarm role } //$DBServer->Delete(); } } catch (Exception $e) { } } else { try { $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS)); if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); $DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } else { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { $DBFarmRole = DBFarmRole::LoadByID($farm_role['id']); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS; } catch (Exception $e) { $this->Logger->error($e->getMessage()); $BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage())); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; } } $BundleTask->Save(); } catch (Exception $e) { $this->Logger->error($e->getMessage()); } break; } }
public function ServerLaunch($FarmRoleID, $IncreaseMaxInstances = false) { try { $DBFarmRole = DBFarmRole::LoadByID($FarmRoleID); $DBFarm = DBFarm::LoadByID($DBFarmRole->FarmID); } catch (Exception $e) { throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID)); } if ($DBFarm->EnvID != $this->Environment->id) { throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID)); } if ($DBFarm->Status != FARM_STATUS::RUNNING) { throw new Exception(sprintf("Farm ID #%s is not running", $DBFarm->ID)); } $this->user->getPermissions()->validate($DBFarm); $this->restrictFarmAccess($DBFarm, Acl::PERM_FARMS_SERVERS); $isSzr = true; $n = $DBFarmRole->GetPendingInstancesCount(); if ($n >= 5 && !$isSzr) { throw new Exception("There are {$n} pending instances. You cannot launch new instances while you have 5 pending ones."); } $response = $this->CreateInitialResponse(); $max_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES); $min_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES); if ($IncreaseMaxInstances) { if ($max_instances < $min_instances + 1) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1, DBFarmRole::TYPE_CFG); } } if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() >= $max_instances) { if ($IncreaseMaxInstances) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1, DBFarmRole::TYPE_CFG); } else { throw new Exception("Max instances limit reached. Use 'IncreaseMaxInstances' parameter or increase max isntances settings in UI"); } } if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() == $min_instances) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $min_instances + 1, DBFarmRole::TYPE_CFG); } $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_MANUALLY_API, $this->user); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Starting new instance (API). ServerID = %s.", $DBServer->serverId), $DBServer->serverId)); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::API)->error($e->getMessage()); } $response->ServerID = $DBServer->serverId; $response->CloudServerID = $DBServer->GetCloudServerID(); return $response; }
public function xLaunchNewServerAction() { $dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId')); $dbFarm = $dbFarmRole->GetFarmObject(); $this->user->getPermissions()->validate($dbFarmRole); if ($dbFarm->Status != FARM_STATUS::RUNNING) { throw new Exception("You can launch servers only on running farms"); } $dbRole = $dbFarmRole->GetRoleObject(); $isSzr = $dbFarmRole->GetRoleObject()->isSupported("0.5"); $pendingInstancesCount = $dbFarmRole->GetPendingInstancesCount(); if ($pendingInstancesCount >= 5 && !$isSzr) { throw new Exception("There are {$pendingInstancesCount} pending instances. You cannot launch new instances while you have 5 pending ones."); } $maxInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES); $minInstances = $dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES); if ($maxInstances < $minInstances + 1) { $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $maxInstances + 1); $warnMsg = sprintf(_("Server successfully launched. The number of running %s instances is equal to maximum instances setting for this role. Maximum Instances setting for role %s has been increased automatically"), $dbRole->name, $dbRole->name); } $runningInstancesCount = $dbFarmRole->GetRunningInstancesCount(); if ($runningInstancesCount + $pendingInstancesCount >= $minInstances) { $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $minInstances + 1); } $serverCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); Scalr::LaunchServer($serverCreateInfo); if ($warnMsg) { $this->response->warning($warnMsg); } else { $this->response->success('Server successfully launched'); } }
function handleWork($farmId) { $DBFarm = DBFarm::LoadByID($farmId); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) { for ($i = 0; $i < 10; $i++) { if ($DBFarmRole->NewRoleID != '') { $this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled."); continue 2; } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) == '0' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { $this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping..."); continue 2; } // Get polling interval in seconds $polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60; $dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME); if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) { $this->logger->info("Polling interval: every {$polling_interval} seconds"); //continue; } // Set Last polling time $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time()); // Get current count of running and pending instances. $this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); $scalingDecision = $scalingManager->makeScalingDecition(); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue 2; } elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instance�s count after scaling resolution the spare instances are running�g for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; // Select instance that will be terminated // // * Instances ordered by uptime (oldest wil be choosen) // * Instance cannot be mysql master // * Choose the one that was rebundled recently while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED))); if ($DBServer->index == 1 && $serversCount > 1) { continue; } } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { /* * We do not want to delete the most recently synced instance. Because of LA fluctuation. * I.e. LA may skyrocket during sync and drop dramatically after sync. */ if ($DBServer->dateLastSync != 0) { $chk_sync_time = $this->db->GetOne("SELECT server_id FROM servers \n\t \t\tWHERE dtlastsync > {$DBServer->dateLastSync} \n\t\t \tAND farm_roleid='{$DBServer->farmRoleId}' AND status != '" . SERVER_STATUS::TERMINATED . "'"); if ($chk_sync_time) { $got_valid_instance = true; } } else { $got_valid_instance = true; } } } if ($DBServer && $got_valid_instance) { $this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $AmazonEC2Client = Scalr_Service_Cloud_Aws::newEc2($DBServer->GetProperty(EC2_SERVER_PROPERTIES::REGION), $DBServer->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), $DBServer->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE)); // Shutdown an instance just before a full hour running $response = $AmazonEC2Client->DescribeInstances($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)); if ($response && $response->reservationSet->item) { $launch_time = strtotime($response->reservationSet->item->instancesSet->item->launchTime); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->reservationSet->item->instancesSet->item->launchTime))); } } // } else { $allow_terminate = true; } if ($allow_terminate) { //Check safe shutdown if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) { $snmpClient = new Scalr_Net_Snmp_Client(); $port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT); $snmpClient->connect($DBServer->remoteIp, $port ? $port : 161, $DBFarm->Hash, null, null, false); $res = $snmpClient->get('1.3.6.1.4.1.36632.6.1'); if ($res != '1') { Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script return '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res))); } } try { Scalr::FireEvent($DBFarm->ID, new BeforeHostTerminateEvent($DBServer, false)); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time()); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { $this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage())); } } } else { $this->logger->warn(sprintf("[FarmID: {$DBFarm->ID}] Scalr unable to determine what instance it should terminate (FarmRoleID: {$DBFarmRole->ID}). Skipping...")); } break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution �need more instances� for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count increase if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name))); continue 2; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=?", array($DBFarm->ID)); if ($fstatus != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time()); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //It must be done for any worker in daemon mode. (Policy engine, PM, etc.. settings shouldn't be cached) \Scalr::getContainer()->warmup(); try { $dbServer = DBServer::LoadByID($request->serverId); if ($dbServer->status == SERVER_STATUS::TEMPORARY) { try { $dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); } catch (Exception $e) { } } else { if ($dbServer->status == SERVER_STATUS::IMPORTING) { $dbServer->Remove(); } else { if ($dbServer->status == SERVER_STATUS::PENDING_LAUNCH) { $account = Scalr_Account::init()->loadById($dbServer->clientId); if ($account->status == Scalr_Account::STATUS_ACTIVE) { \Scalr::LaunchServer(null, $dbServer); } } } } } catch (Exception $e) { $this->getLogger()->error("Server: %s, manager failed with exception: %s", $request->serverId, $e->getMessage()); } return $request; }
/** * Farm launched * * @param FarmLaunchedEvent $event */ public function OnFarmLaunched(FarmLaunchedEvent $event) { $DBFarm = DBFarm::LoadByID($this->FarmID); // TODO: Refactoting -> Move to DBFarm class $this->DB->Execute("UPDATE farms SET status=?, dtlaunched=NOW() WHERE id=?", array(FARM_STATUS::RUNNING, $this->FarmID)); $governance = new Scalr_Governance($DBFarm->EnvID); if ($governance->isEnabled(Scalr_Governance::GENERAL_LEASE) && $DBFarm->GetSetting(DBFarm::SETTING_LEASE_STATUS)) { $dt = new DateTime(); $dt->add(new DateInterval('P' . intval($governance->getValue(Scalr_Governance::GENERAL_LEASE, 'defaultLifePeriod')) . 'D')); $DBFarm->SetSetting(DBFarm::SETTING_LEASE_EXTEND_CNT, 0); $DBFarm->SetSetting(DBFarm::SETTING_LEASE_TERMINATE_DATE, $dt->format('Y-m-d H:i:s')); $DBFarm->SetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND, ''); } $roles = $DBFarm->GetFarmRoles(); foreach ($roles as $dbFarmRole) { if ($dbFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) && !$DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { $scalingManager = new Scalr_Scaling_Manager($dbFarmRole); $scalingDecision = $scalingManager->makeScalingDecition(); if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { $ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo, null, true, "Farm launched", isset($event->userId) ? $event->userId : null); $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $dbFarmRole->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } } }
function handleWork($farmId) { $DBFarm = DBFarm::LoadByID($farmId); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) { for ($i = 0; $i < 10; $i++) { if ($DBFarmRole->NewRoleID != '') { $this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled."); continue 2; } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) != '1' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { $this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping..."); continue 2; } // Get polling interval in seconds $polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60; $dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME); if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) { $this->logger->info("Polling interval: every {$polling_interval} seconds"); continue; } // Set Last polling time $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time(), DBFarmRole::TYPE_LCL); // Get current count of running and pending instances. $this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); $scalingDecision = $scalingManager->makeScalingDecition(); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue 2; } elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instance�s count after scaling resolution the spare instances are running�g for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; // Select instance that will be terminated // // * Instances ordered by uptime (oldest wil be choosen) // * Instance cannot be mysql master // * Choose the one that was rebundled recently while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING)))); if ($DBServer->index == 1 && $serversCount > 1) { continue; } } if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) { continue; } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { // We do not want to delete the most recently synced instance. Because of LA fluctuation. // I.e. LA may skyrocket during sync and drop dramatically after sync. if ($DBServer->dateLastSync != 0) { $chk_sync_time = $this->db->GetOne("\n SELECT server_id FROM servers\n WHERE dtlastsync > {$DBServer->dateLastSync}\n AND farm_roleid='{$DBServer->farmRoleId}'\n AND status NOT IN('" . SERVER_STATUS::TERMINATED . "', '" . SERVER_STATUS::TROUBLESHOOTING . "')\n LIMIT 1\n "); if ($chk_sync_time) { $got_valid_instance = true; } } else { $got_valid_instance = true; } } } if ($DBServer && $got_valid_instance) { $this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $aws = $DBServer->GetEnvironmentObject()->aws($DBServer); // Shutdown an instance just before a full hour running if (!$DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_IGNORE_FULL_HOUR)) { $response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0); if ($response && count($response->instancesSet)) { $launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp(); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')))); } } } else { $allow_terminate = true; } //Releases memory $DBServer->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } else { $allow_terminate = true; } if ($allow_terminate) { //Check safe shutdown if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) { if ($DBServer->IsSupported('0.11.3')) { try { $port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_API_PORT); if (!$port) { $port = 8010; } $szrClient = Scalr_Net_Scalarizr_Client::getClient($DBServer, Scalr_Net_Scalarizr_Client::NAMESPACE_SYSTEM, $port); $res = $szrClient->callAuthShutdownHook(); } catch (Exception $e) { $res = $e->getMessage(); } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled, but not supported by scalarizr installed on server '%s'. Ignoring.", $DBServer->serverId))); } if ($res != '1') { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res))); break; } } try { $DBServer->terminate('SCALING_DOWN', false); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { $this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage())); } } } else { $this->logger->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $DBFarm->ID, $DBFarmRole->ID)); } break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution �need more instances� for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name))); continue 2; } } // end Timeout instance's count increase //Check DBMsr. Do not start slave during slave2master process $isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior(); if ($isDbMsr) { if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) { $runningServers = $DBFarmRole->GetRunningInstancesCount(); if ($runningServers > 0) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves"))); continue 2; } else { $DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL); } } } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name))); continue 2; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($DBFarm->ID)); if ($fstatus != FARM_STATUS::RUNNING) { $this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it."); return; } $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Scaling up"); $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } } }
/** * Farm launched * * @param FarmLaunchedEvent $event */ public function OnFarmLaunched(FarmLaunchedEvent $event) { $DBFarm = DBFarm::LoadByID($this->FarmID); // TODO: Refactoting -> Move to DBFarm class $this->DB->Execute("UPDATE farms SET status=?, dtlaunched=NOW() WHERE id=?", array(FARM_STATUS::RUNNING, $this->FarmID)); $roles = $DBFarm->GetFarmRoles(); foreach ($roles as $dbFarmRole) { $scalingManager = new Scalr_Scaling_Manager($dbFarmRole); $scalingDecision = $scalingManager->makeScalingDecition(); if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { $ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo, null, true); $dbFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time()); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $dbFarmRole->GetRoleObject()->name, $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } }
public function StartThread($bundle_task_info) { $db = \Scalr::getDb(); // Reconfigure observers; Scalr::ReconfigureObservers(); $BundleTask = BundleTask::LoadById($bundle_task_info['id']); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); } catch (\Scalr\Exception\ServerNotFoundException $e) { if (!$BundleTask->snapshotId) { $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED; $BundleTask->setDate('finished'); $BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId); $BundleTask->Save(); return; } } catch (Exception $e) { //$this->Logger->error($e->getMessage()); } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION: $conn = @fsockopen($DBServer->getSzrHost(), $DBServer->getPort(DBServer::PORT_CTRL), $errno, $errstr, 10); if ($conn) { $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1); $BundleTask->Log("Outbound connection successfully established. Awaiting user action: prebuild automation selection"); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION; $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); $BundleTask->Save(); } else { $errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $requestHost, $ctrlPort, $errstr); $errMsg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR); if (!$errMsg || $errstr != $errMsg) { $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr); $BundleTask->Log("{$errstr} Will try again in a few minutes."); } } exit; break; case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION: //NOTHING TO DO; exit; break; case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $BundleTask->setDate('started'); case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: if (!PlatformFactory::NewPlatform($DBServer->platform)->GetServerID($DBServer)) { $BundleTask->Log(sprintf(_("Waiting for temporary server"))); exit; } if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } // IF server is in pensing state $status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer); if ($status->isPending()) { $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); $BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName())); exit; } elseif ($status->isTerminated()) { $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } break; } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $DBServer->remoteIp = $ips['remoteIp']; $DBServer->localIp = $ips['localIp']; $DBServer->save(); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV; $BundleTask->save(); $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: $BundleTask->Log(sprintf(_("Initializing SSH2 session to the server"))); if ($DBServer->platform == SERVER_PLATFORMS::IDCF && !$DBServer->remoteIp) { try { $BundleTask->Log("Creating port forwarding rules to be able to connect to the server by SSH"); $environment = $DBServer->GetEnvironmentObject(); $cloudLocation = $DBServer->GetCloudLocation(); $platform = PlatformFactory::NewPlatform($DBServer->platform); $sharedIpId = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP_ID . ".{$cloudLocation}", $environment, false); $sharedIp = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP . ".{$cloudLocation}", $environment, false); $BundleTask->Log("Shared IP: {$sharedIp}"); $cs = $environment->cloudstack($DBServer->platform); // Create port forwarding rules for scalarizr $port = $platform->getConfigVariable(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false); if (!$port) { $port1 = 30000; $port2 = 30001; $port3 = 30002; $port4 = 30003; } else { $port1 = $port + 1; $port2 = $port1 + 1; $port3 = $port2 + 1; $port4 = $port3 + 1; } $virtualmachineid = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID); $result2 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8014, 'protocol' => "udp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid)); $result1 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8013, 'protocol' => "tcp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid)); $result3 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8010, 'protocol' => "tcp", 'publicport' => $port3, 'virtualmachineid' => $virtualmachineid)); $result4 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8008, 'protocol' => "tcp", 'publicport' => $port2, 'virtualmachineid' => $virtualmachineid)); $result5 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 22, 'protocol' => "tcp", 'publicport' => $port4, 'virtualmachineid' => $virtualmachineid)); $DBServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4)); $DBServer->remoteIp = $sharedIp; $DBServer->Save(); $platform->setConfigVariable(array(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false); } catch (Exception $e) { $BundleTask->Log("Unable to create port-forwarding rules: {$e->getMessage()}"); } exit; } if ($DBServer->platform == SERVER_PLATFORMS::ECS && !$DBServer->remoteIp) { $BundleTask->Log(sprintf(_("Server doesn't have public IP. Assigning..."))); $osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION)); $ports = $osClient->network->ports->list(); foreach ($ports as $port) { if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $serverNetworkPort = $port->id; break; } } $ips = $osClient->network->floatingIps->list(); //Check free existing IP $ipAssigned = false; $ipAddress = false; $ipId = false; $ipInfo = false; foreach ($ips as $ip) { if ($ip->port_id && $ip->port_id == $serverNetworkPort) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipAssigned = true; $ipInfo = $ip; break; } if (!$ip->fixed_ip_address && !$ipAddress) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipInfo = $ip; } } if (!$ipAssigned) { if (!$serverNetworkPort) { $BundleTask->Log("Unable to identify network port of instance"); exit; } else { if (!$ipAddress) { $networks = $osClient->network->listNetworks(); foreach ($networks as $network) { if ($network->{"router:external"} == true) { $publicNetworkId = $network->id; } } if (!$publicNetworkId) { $BundleTask->Log("Unable to identify public network to allocate"); exit; } else { $ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort); $ipAddress = $ip->floating_ip_address; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); $BundleTask->Log("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}"); } } else { $BundleTask->Log("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")"); $osClient->network->floatingIps->update($ipId, $serverNetworkPort); } } } else { $BundleTask->Log("IP: {$ipAddress} already assigned"); } if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } exit; } try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $BundleTask->Log(sprintf(_("Created SSH session. Username: %s"), $ssh2Client->getLogin())); //Prepare script $BundleTask->Log(sprintf(_("Uploading builder scripts..."))); $behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR); try { if ($DBServer->isOpenstack()) { $platform = SERVER_PLATFORMS::OPENSTACK; } else { $platform = $DBServer->platform; } $baseUrl = rtrim(\Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host'), '/'); $options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $DBServer->envId, 'region' => $DBServer->GetCloudLocation(), 'scalr-id' => SCALR_ID); $command = 'scalarizr --import -y'; foreach ($options as $k => $v) { $command .= sprintf(' -o %s=%s', $k, $v); } if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') { $recipes = 'mysql=percona'; } else { $recipes = ''; } $scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH); $scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl"); /* %CHEF_SERVER_URL% %CHEF_VALIDATOR_NAME% %CHEF_VALIDATOR_KEY% %CHEF_ENVIRONMENT% %CHEF_ROLE_NAME% */ $chefServerId = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID); if ($chefServerId) { $chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId)); $chefServerInfo['v_auth_key'] = $this->crypto->decrypt($chefServerInfo['v_auth_key'], $this->cryptoKey); } $scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents); if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) { throw new Exception("Cannot upload script"); } /* $BundleTask->Log(sprintf(_("Uploading chef recipes..."))); if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) { throw new Exception("Cannot upload chef recipes"); } */ } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $BundleTask->Log("Launching role builder routines on server"); $ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh"); // For CGE we need to use sudo if ($BundleTask->platform == SERVER_PLATFORMS::GCE || $BundleTask->osFamily == 'amazon') { $shell = $ssh2Client->getShell(); @stream_set_blocking($shell, true); @stream_set_timeout($shell, 5); @fwrite($shell, "sudo touch /var/log/role-builder-output.log 2>&1" . PHP_EOL); $output = @fgets($shell, 4096); $BundleTask->Log("Verbose 1: {$output}"); @fwrite($shell, "sudo chmod 0666 /var/log/role-builder-output.log 2>&1" . PHP_EOL); $output2 = @fgets($shell, 4096); $BundleTask->Log("Verbose 2: {$output2}"); @fwrite($shell, "sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &" . PHP_EOL); $output3 = @fgets($shell, 4096); $BundleTask->Log("Verbose 3: {$output3}"); sleep(5); $meta = stream_get_meta_data($shell); $BundleTask->Log(sprintf("Verbose (Meta): %s", json_encode($meta))); $i = 4; if ($meta['eof'] == false && $meta['unread_bytes'] != 0) { $output4 = @fread($shell, $meta['unread_bytes']); $BundleTask->Log("Verbose {$i}: {$output4}"); $meta = stream_get_meta_data($shell); $BundleTask->Log(sprintf("Verbose (Meta): %s", json_encode($meta))); } @fclose($shell); /* $r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log"); $BundleTask->Log("1: {$r1} ({$ssh2Client->stdErr})"); $r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log"); $BundleTask->Log("2: {$r2} ({$ssh2Client->stdErr})"); $r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); $BundleTask->Log("3: {$r3} ({$ssh2Client->stdErr})"); */ } else { $ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE; $BundleTask->save(); break; case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $log = $ssh2Client->getFile('/var/log/role-builder-output.log'); $log_lines = explode("\r\n", $log); $last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE); while ($msg = trim(array_shift($log_lines))) { if (substr($msg, -1, 1) != ']') { continue; } if ($last_msg) { if ($msg != $last_msg) { continue; } elseif ($msg == $last_msg) { $last_msg = null; continue; } } if (stristr($msg, '[ Failed ]')) { $stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log'); $BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog)); $BundleTask->SnapshotCreationFailed($msg); } else { $BundleTask->Log($msg); $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg); } } //Read /var/log/role-builder-output.log break; case SERVER_SNAPSHOT_CREATION_STATUS::PENDING: try { $platformModule = PlatformFactory::NewPlatform($BundleTask->platform); $platformModule->CreateServerSnapshot($BundleTask); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage()); $BundleTask->SnapshotCreationFailed($e->getMessage()); } break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING: $addedTime = strtotime($BundleTask->dateAdded); if ($addedTime + 3600 < time()) { $BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time."); } break; case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS: PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask); break; case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS: $r_farm_roles = array(); $BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType)); try { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); } catch (Exception $e) { if (stristr($e->getMessage(), "not found in database")) { $BundleTask->SnapshotCreationFailed("Farm was removed before task was finished"); } return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { try { $r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); } catch (Exception $e) { } } elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=? AND new_role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { try { $r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']); } catch (Exception $e) { } } } $update_farm_dns_zones = array(); $completed_roles = 0; foreach ($r_farm_roles as $DBFarmRole) { if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) { $BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); $completed_roles++; } else { $servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)); $BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. " . "Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); if (count($servers) == 0) { $DBFarmRole->RoleID = $DBFarmRole->NewRoleID; $DBFarmRole->NewRoleID = null; $DBFarmRole->Save(); $update_farm_dns_zones[$DBFarmRole->FarmID] = 1; $completed_roles++; } else { $metaData = $BundleTask->getSnapshotDetails(); foreach ($servers as $server) { try { $DBServer = DBServer::LoadByID($server['server_id']); } catch (Exception $e) { //TODO: continue; } if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) { $DBServer->roleId = $BundleTask->roleId; $DBServer->Save(); if ($metaData['noServersReplace']) { $BundleTask->Log(sprintf("'Do not replace servers' option was checked. " . "Server '%s' won't be replaced to new image.", $DBServer->serverId)); } else { $BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, " . "already has all modifications. No need to replace it.", $DBServer->serverId)); } } else { if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?,?) LIMIT 1", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TROUBLESHOOTING, SERVER_STATUS::SUSPENDED))) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID); $nDBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_REPLACE_SERVER_FROM_SNAPSHOT, !empty($BundleTask->createdById) ? $BundleTask->createdById : null); $nDBServer->replaceServerID = $DBServer->serverId; $nDBServer->Save(); $BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId)); } } } } } } if ($completed_roles == count($r_farm_roles)) { $BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { if ($DBServer->status == SERVER_STATUS::IMPORTING) { $DBServer->Remove(); } elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) { $BundleTask->Log("Terminating temporary server"); $DBServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); $BundleTask->Log("Termination request has been sent"); } } catch (Exception $e) { $BundleTask->Log("Warning: {$e->getMessage()}"); } $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Save(); $BundleTask->createImageEntity(); } try { if (count($update_farm_dns_zones) != 0) { foreach ($update_farm_dns_zones as $farm_id => $v) { $dnsZones = DBDNSZone::loadByFarmId($farm_id); foreach ($dnsZones as $dnsZone) { if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) { $dnsZone->updateSystemRecords(); $dnsZone->save(); } } } } } catch (Exception $e) { $this->Logger->fatal("DNS ZONE: {$e->getMessage()}"); } break; case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE: try { if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $saveOldRole = false; try { $dbRole = DBRole::loadById($DBServer->roleId); if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) { $saveOldRole = true; } } catch (Exception $e) { //NO OLD ROLE } if ($dbRole && $saveOldRole) { if ($DBServer) { $new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer); } else { $new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999); } $dbRole->name = $new_role_name; $BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name)); $dbRole->save(); } else { //TODO: //$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})"); } } if ($BundleTask->object == BundleTask::BUNDLETASK_OBJECT_ROLE) { try { $DBRole = DBRole::createFromBundleTask($BundleTask); } catch (Exception $e) { $BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again."); return; } } else { // roleName is empty, just create image $BundleTask->createImageEntity(); } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) { $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); if ($DBServer->status == SERVER_STATUS::IMPORTING) { if ($DBServer->farmId) { // Create DBFarmRole object // TODO: create DBFarm role } } } catch (Exception $e) { } } else { try { $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS)); if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); $DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } else { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { $DBFarmRole = DBFarmRole::LoadByID($farm_role['id']); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS; } catch (Exception $e) { $this->Logger->error($e->getMessage()); $BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage())); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; } } if ($BundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) { try { if ($DBServer->status == SERVER_STATUS::IMPORTING) { $DBServer->Remove(); } elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) { $BundleTask->Log("Terminating temporary server"); $DBServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); $BundleTask->Log("Termination request has been sent"); } } catch (Exception $e) { $BundleTask->Log("Warning: {$e->getMessage()}"); } } $BundleTask->Save(); $BundleTask->createImageEntity(); } catch (Exception $e) { $this->Logger->error($e->getMessage()); } break; } }
public function ServerLaunch($FarmRoleID, $IncreaseMaxInstances = false) { try { $DBFarmRole = DBFarmRole::LoadByID($FarmRoleID); $DBFarm = DBFarm::LoadByID($DBFarmRole->FarmID); } catch (Exception $e) { throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID)); } if ($DBFarm->EnvID != $this->Environment->id) { throw new Exception(sprintf("Farm Role ID #%s not found", $FarmRoleID)); } //TODO: Remove this limitation /* $isSzr = $dbFarmRole->GetRoleObject()->isSupported("0.5"); if ($retval == Scalr_Scaling_Decision::UPSCALE && ($dbFarmRole->GetPendingInstancesCount() > 5 || !$isSzr)) */ $isSzr = $DBFarmRole->GetRoleObject()->isSupported("0.5"); $n = $DBFarmRole->GetPendingInstancesCount(); if ($n >= 5 && !$isSzr) { throw new Exception("There are {$n} pending instances. You cannot launch new instances while you have 5 pending ones."); } $response = $this->CreateInitialResponse(); $max_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES); $min_instances = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES); if ($IncreaseMaxInstances) { if ($max_instances < $min_instances + 1) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1); } } if ($DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount() >= $max_instances) { if ($IncreaseMaxInstances) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES, $max_instances + 1); } else { throw new Exception("Max instances limit reached. Use 'IncreaseMaxInstances' parameter or increase max isntances settings in UI"); } } if ($DBFarmRole->GetRunningInstancesCount() < $min_instances || $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_MAX_INSTANCES) < $min_instances) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_MIN_INSTANCES, $min_instances + 1); } $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = Scalr::LaunchServer($ServerCreateInfo); Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Starting new instance (API). ServerID = %s.", $DBServer->serverId))); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::API)->error($e->getMessage()); } $response->ServerID = $DBServer->serverId; return $response; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { try { $dbServer = DBServer::LoadByID($request->serverId); if ($dbServer->status == SERVER_STATUS::TEMPORARY) { try { $dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); } catch (Exception $e) { } } else { if ($dbServer->status == SERVER_STATUS::IMPORTING) { $dbServer->Remove(); } else { if ($dbServer->status == SERVER_STATUS::PENDING_LAUNCH) { $account = Scalr_Account::init()->loadById($dbServer->clientId); if ($account->status == Scalr_Account::STATUS_ACTIVE) { \Scalr::LaunchServer(null, $dbServer); } } } } } catch (Exception $e) { $this->getLogger()->error("Server: %s, manager failed with exception: %s", $request->serverId, $e->getMessage()); } return $request; }
private function cleanup() { // Check that time has come to cleanup dead servers $doCleanup = false; sem_acquire($this->cleanupSem); try { if (time() - (int) $this->lastCleanup->get(0) >= $this->cleanupInterval) { $doCleanup = true; $this->lastCleanup->put(0, time()); } } catch (Exception $e) { sem_release($this->cleanupSem); } sem_release($this->cleanupSem); if ($doCleanup) { $this->logger->info("Cleanup dead servers"); try { $terminated_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=? AND (UNIX_TIMESTAMP(dtshutdownscheduled)+3600 < UNIX_TIMESTAMP(NOW()) OR dtshutdownscheduled IS NULL)", array(SERVER_STATUS::TERMINATED)); foreach ($terminated_servers as $ts) { DBServer::LoadByID($ts['server_id'])->Remove(); } $p_terminated_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=? AND role_id='0' AND farm_id IS NULL", array(SERVER_STATUS::PENDING_TERMINATE)); foreach ($p_terminated_servers as $ts) { DBServer::LoadByID($ts['server_id'])->Remove(); } $importing_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status IN(?,?) AND UNIX_TIMESTAMP(dtadded)+86400 < UNIX_TIMESTAMP(NOW())", array(SERVER_STATUS::IMPORTING, SERVER_STATUS::TEMPORARY)); foreach ($importing_servers as $ts) { DBServer::LoadByID($ts['server_id'])->Remove(); } $pending_launch_servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status=?", array(SERVER_STATUS::PENDING_LAUNCH)); try { foreach ($pending_launch_servers as $ts) { $DBServer = DBServer::LoadByID($ts['server_id']); $account = Scalr_Account::init()->loadById($DBServer->clientId); if ($account->status == Scalr_Account::STATUS_ACTIVE) { Scalr::LaunchServer(null, $DBServer); } } } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Can't load server with ID #'%s'", $ts['server_id'], $e->getMessage())); } } catch (Exception $e) { $this->logger->fatal("Poller::cleanup failed: {$e->getMessage()}"); } } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if (!isset($request->farmRoleId)) { //This is the farm with synchronous launch of roles try { $DBFarm = DBFarm::LoadByID($request->farmId); if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID); return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId); throw $e; } //Gets the list of the roles $list = $DBFarm->GetFarmRoles(); } else { //This is asynchronous lauhch try { $DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId); if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) { //We don't need to handle inactive farms return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId); throw $e; } $list = [$DBFarmRole]; } $this->getLogger()->debug("Processing %s FarmRoles", count($list)); foreach ($list as $DBFarmRole) { // Set Last polling time $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL); $disabledScaling = false; if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') { if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { // For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running) // When 1 instance is already running, the rest is fully manual $roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount(); if ($roleTotalInstances != 0) { $disabledScaling = true; } } else { if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { $disabledScaling = true; } } if ($disabledScaling) { $this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias); continue; } } $farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name; // Get current count of running and pending instances. $this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); //Replacing the logger $scalingManager->logger = $this->getLogger(); $scalingDecision = $scalingManager->makeScalingDecision(); $scalingDecisionDetails = $scalingManager->decisonInfo; $this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails)); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue; } else { if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instances count after scaling resolution the spare instances are running for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null)); continue; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; $ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR); $useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN); $isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ); // Select instance that will be terminated // // Instances ordered by uptime (oldest wil be choosen) // Instance cannot be mysql master // Choose the one that was rebundled recently $DBServer = null; while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($isRabbitMQ) { $serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]); if ($DBServer->index == 1 && $serverExists) { continue; } } if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) { continue; } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { $got_valid_instance = true; } //Check safe shutdown if ($useSafeShutdown == 1) { try { $res = $DBServer->scalarizr->system->callAuthShutdownHook(); } catch (Exception $e) { $res = $e->getMessage(); } if ($res != '1') { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); $got_valid_instance = false; } } } // end while if ($DBServer !== null && $got_valid_instance) { $this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $aws = $DBServer->GetEnvironmentObject()->aws($DBServer); // Shutdown an instance just before a full hour running if (!$ignoreFullHour) { $response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0); if ($response && count($response->instancesSet)) { $launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp(); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } } else { $allow_terminate = true; } //Releases memory $DBServer->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } else { $allow_terminate = true; } if ($allow_terminate) { $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } try { if ($terminateStrategy == 'terminate') { $DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } else { $DBServer->suspend('SCALING_DOWN', false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } catch (Exception $e) { $this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId)); } } } else { $this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID)); } //break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution 'need more instances' for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias)); continue; } } // end Timeout instance's count increase //Check DBMsr. Do not start slave during slave2master process $isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior(); if ($isDbMsr) { if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) { $runningServers = $DBFarmRole->GetRunningInstancesCount(); if ($runningServers > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves"))); continue; } else { $DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL); } } } if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName))); continue; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId)); if ($fstatus != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it."); return; } $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } $suspendedServer = null; if ($terminateStrategy == 'suspend') { $suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED)); if (count($suspendedServers) > 0) { $suspendedServer = array_shift($suspendedServers); } } if ($terminateStrategy == 'suspend' && $suspendedServer) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId)); } if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } catch (Exception $e) { \Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } else { $platform = PlatformFactory::NewPlatform($suspendedServer->platform); $platform->ResumeServer($suspendedServer); } } } } return true; }
/** * {@inheritdoc} * @see \Scalr\Observer\AbstractEventObserver::OnFarmLaunched() */ public function OnFarmLaunched(FarmLaunchedEvent $event) { $DBFarm = DBFarm::LoadByID($this->FarmID); $this->DB->Execute("UPDATE farms SET status = ?, dtlaunched = NOW() WHERE id = ? LIMIT 1", [FARM_STATUS::RUNNING, $this->FarmID]); $this->getContainer()->auditlogger->log('farm.launch', $DBFarm, $event->auditLogExtra); $governance = new Scalr_Governance($DBFarm->EnvID); if ($governance->isEnabled(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE) && $DBFarm->GetSetting(Entity\FarmSetting::LEASE_STATUS)) { $dt = new DateTime(); $dt->add(new DateInterval('P' . intval($governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'defaultLifePeriod')) . 'D')); $DBFarm->SetSetting(Entity\FarmSetting::LEASE_EXTEND_CNT, 0); $DBFarm->SetSetting(Entity\FarmSetting::LEASE_TERMINATE_DATE, $dt->format('Y-m-d H:i:s')); $DBFarm->SetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND, ''); } $roles = $DBFarm->GetFarmRoles(); foreach ($roles as $dbFarmRole) { if ($dbFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) && !$DBFarm->GetSetting(Entity\FarmSetting::EC2_VPC_ID)) { $scalingManager = new Scalr_Scaling_Manager($dbFarmRole); $scalingDecision = $scalingManager->makeScalingDecision(); if ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { $ServerCreateInfo = new ServerCreateInfo($dbFarmRole->Platform, $dbFarmRole); try { $DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, true, DBServer::LAUNCH_REASON_FARM_LAUNCHED, isset($event->userId) ? $event->userId : null); $dbFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); $role = $dbFarmRole->GetRoleObject(); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", !empty($DBFarm->Name) ? $DBFarm->Name : null, !empty($role->name) ? $role->name : null, !empty($DBServer->serverId) ? $DBServer->serverId : null), !empty($DBServer->serverId) ? $DBServer->serverId : null, !empty($DBServer->envId) ? $DBServer->envId : null, !empty($DBServer->farmRoleId) ? $DBServer->farmRoleId : null)); } catch (Exception $e) { \Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } } } }