Checks wheter specified cloud is OpenStack based
public function deleteAction()
{
$this->request->defineParams(array('sshKeyId' => array('type' => 'int')));
$sshKey = Scalr_SshKey::init()->loadById($this->getParam('sshKeyId'));
$this->user->getPermissions()->validate($sshKey);
if ($sshKey->type == Scalr_SshKey::TYPE_GLOBAL) {
if ($sshKey->platform == SERVER_PLATFORMS::EC2) {
$aws = $this->getEnvironment()->aws($sshKey->cloudLocation);
$aws->ec2->keyPair->delete($sshKey->cloudKeyName);
$sshKey->delete();
$this->response->success();
} elseif (PlatformFactory::isOpenstack($sshKey->platform)) {
$os = $this->getEnvironment()->openstack($sshKey->platform, $sshKey->cloudLocation);
try {
$os->servers->keypairs->delete($sshKey->cloudKeyName);
} catch (Exception $e) {
}
$sshKey->delete();
$this->response->success();
} else {
$sshKey->delete();
}
} else {
//TODO:
}
$this->response->success("SSH key successfully removed");
}
/**
* Gets a new Instance of the adapter
*
* @param string|CloudCredentials|object $name The name of the adapter, or CloudCredentials entity, or cloud credentials data
*
* @return ApiEntityAdapter Returns the instance of cloud credentials adapter
*
* @throws ApiErrorException
*/
public function adapter($name, array $transform = null)
{
if (is_object($name)) {
//
$property = $name instanceof $this->entityClass ? static::$entityDescriminator : static::$objectDiscriminator;
$value = empty($transform) ? $name->{$property} : $transform[$name->{$property}];
switch (true) {
case PlatformFactory::isOpenstack($value, true):
$value = SERVER_PLATFORMS::OPENSTACK;
break;
case PlatformFactory::isCloudstack($value):
$value = SERVER_PLATFORMS::CLOUDSTACK;
break;
case PlatformFactory::isRackspace($value):
$value = SERVER_PLATFORMS::RACKSPACE;
break;
}
if (!isset(static::$inheritanceMap[$value])) {
throw new ApiErrorException(400, ErrorMessage::ERR_INVALID_VALUE, "Unknown cloud '{$value}'");
}
$class = empty(static::$inheritanceMap) ? $value : static::$inheritanceMap[$value];
$name = empty(static::$inheritedNamespace) ? $class : static::$inheritedNamespace . "\\{$class}";
}
return parent::adapter($name);
}
/**
* Constructor
*
* @param string $platform Platform
* @param DBFarmRole $DBFarmRole optional Farm Role object
* @param int $index optional Server index within the Farm Role scope
* @param string $role_id optional Identifier of the Role
*/
public function __construct($platform, DBFarmRole $DBFarmRole = null, $index = null, $role_id = null)
{
$this->platform = $platform;
$this->dbFarmRole = $DBFarmRole;
$this->index = $index;
$this->roleId = $role_id === null ? $this->dbFarmRole->RoleID : $role_id;
if ($DBFarmRole) {
$this->envId = $DBFarmRole->GetFarmObject()->EnvID;
}
//Refletcion
$Reflect = new ReflectionClass(DBServer::$platformPropsClasses[$this->platform]);
foreach ($Reflect->getConstants() as $k => $v) {
$this->platformProps[] = $v;
}
if ($DBFarmRole) {
if (PlatformFactory::isOpenstack($this->platform)) {
$this->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
} elseif (PlatformFactory::isCloudstack($this->platform)) {
$this->SetProperties(array(CLOUDSTACK_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
} else {
switch ($this->platform) {
case SERVER_PLATFORMS::GCE:
$this->SetProperties(array(GCE_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
break;
case SERVER_PLATFORMS::EC2:
$this->SetProperties(array(EC2_SERVER_PROPERTIES::REGION => $DBFarmRole->CloudLocation));
break;
}
}
}
$this->SetProperties(array(SERVER_PROPERTIES::SZR_VESION => '0.20.0'));
}
/**
* @param string $platform
* @param string $cloudLocation
* @throws Exception
*/
public function xGetInstanceTypesAction($platform, $cloudLocation = null)
{
if (!in_array($platform, $this->getEnvironment()->getEnabledPlatforms())) {
throw new Exception(sprintf('Platform "%s" is not enabled', $platform));
}
$p = PlatformFactory::NewPlatform($platform);
if (PlatformFactory::isOpenstack($platform) && !$cloudLocation) {
$locations = $p->getLocations($this->getEnvironment());
$cloudLocation = @array_pop(@array_keys($locations));
}
$data = [];
foreach ($p->getInstanceTypes($this->getEnvironment(), $cloudLocation, true) as $id => $value) {
$data[] = array_merge(['id' => (string) $id], $value);
}
$this->response->data(array('data' => $data));
}
/**
* Decode params from href depending on type of OS
*
* Possible formats (branch is optioanl):
* for windows: /{repo}/[{branch}/]install_scalarizr.ps1
* for linux: /{repo}/{platform}/[{branch}]/install_scalarizr.sh
*
* @param array $params
* @param bool $isLinux
* @return array
*/
protected function parseInstallScriptArgs($params, $isLinux = true)
{
$result = ['repo' => '', 'platform' => '', 'repoUrls' => []];
$repo = array_shift($params);
$platform = $isLinux ? array_shift($params) : '';
array_pop($params);
// pop script name from the end of href
$branch = implode('/', $params);
$repos = $this->getContainer()->config('scalr.scalarizr_update.' . ($branch ? 'devel_repos' : 'repos'));
if (in_array($repo, array_keys($repos))) {
if ($branch) {
// strip illegal chars
$branch = preg_replace('/[^A-Za-z\\/0-9_.-]/', '', $branch);
if ($repo !== 'snapshot') {
// for snapshot don't cut "." (dot)
$branch = str_replace(array(".", '/'), array('', '-'), $branch);
}
}
$repoUrls = $repos[$repo];
if ($branch) {
foreach ($repoUrls as $key => &$url) {
$url = sprintf($url, $branch);
}
}
if ($isLinux) {
if (in_array($platform, $this->getContainer()->config('scalr.allowed_clouds'))) {
if (PlatformFactory::isOpenstack($platform)) {
$platform = SERVER_PLATFORMS::OPENSTACK;
} else {
if (PlatformFactory::isCloudstack($platform)) {
$platform = SERVER_PLATFORMS::CLOUDSTACK;
}
}
$result['platform'] = $platform;
$result['repo'] = $repo;
$result['repoUrls'] = $repoUrls;
}
} else {
$result['repo'] = $repo;
$result['repoUrls'] = $repoUrls;
}
}
return $result;
}
public static function onCreateBackupResult(Scalr_Messaging_Msg $message, DBServer $dbServer)
{
$dbFarmRole = $dbServer->GetFarmRoleObject();
$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_LAST_TS, time(), DBFarmRole::TYPE_LCL);
$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_IS_RUNNING, 0, DBFarmRole::TYPE_LCL);
//$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_SERVER_ID, "");
if (PlatformFactory::isOpenstack($dbServer->platform)) {
$provider = 'cf';
} else {
switch ($dbServer->platform) {
case SERVER_PLATFORMS::EC2:
$provider = 's3';
break;
case SERVER_PLATFORMS::GCE:
$provider = 'gcs';
break;
default:
$provider = 'unknown';
break;
}
}
$backup = Scalr_Db_Backup::init();
$backup->service = $message->dbType;
$backup->platform = $dbServer->platform;
$backup->provider = $provider;
$backup->envId = $dbServer->envId;
$backup->farmId = $dbServer->farmId;
$backup->cloudLocation = $dbServer->GetCloudLocation();
$backup->status = Scalr_Db_Backup::STATUS_AVAILABLE;
$total = 0;
if (isset($message->backupParts) && is_array($message->backupParts)) {
foreach ($message->backupParts as $item) {
if (is_object($item) && $item->size) {
$backup->addPart(str_replace(array("s3://", "cf://", "gcs://", "swift://"), array("", "", "", ""), $item->path), $item->size);
$total = $total + (int) $item->size;
} else {
$backup->addPart(str_replace(array("s3://", "cf://", "gcs://", "swift://"), array("", "", "", ""), $item), 0);
}
}
}
$backup->size = $total;
$backup->save();
}
protected function run1($stage)
{
foreach ($this->db->GetAll("SELECT id FROM client_environments") as $row) {
try {
$environment = Scalr_Environment::init()->loadById($row['id']);
} catch (Exception $e) {
continue;
}
foreach ($this->getEnabledPlatforms($environment) as $platform) {
if ($platform == SERVER_PLATFORMS::EC2 || PlatformFactory::isOpenstack($platform)) {
$paramName = $platform == SERVER_PLATFORMS::EC2 ? 'aws.tags' : 'openstack.tags';
$nameTagValue = null;
$row = $this->db->GetRow("SELECT * FROM `governance` WHERE env_id = ? AND category = ? AND name = ?", array($environment->id, $platform, $paramName));
$newValue = ['value' => $this->deprecatedTags, 'allow_additional_tags' => 1];
if ($row) {
if ($row['enabled'] == 1) {
$value = json_decode($row['value'], true);
foreach ((array) $value['value'] as $k => $v) {
if ($platform == SERVER_PLATFORMS::EC2 && $k == 'Name') {
$nameTagValue = $v;
continue;
}
if (!isset($newValue['value'][$k])) {
$newValue['value'][$k] = $v;
}
}
$newValue['allow_additional_tags'] = $value['allow_additional_tags'] == 1 ? 1 : 0;
}
}
$newValue = json_encode($newValue);
$this->db->Execute("\n INSERT INTO `governance`\n SET `env_id` = ?,\n `category` = ?,\n `name` = ?,\n `value` = ?,\n `enabled` = ?\n ON DUPLICATE KEY UPDATE\n `value` = ?,\n `enabled` = ?\n ", array($environment->id, $platform, $paramName, $newValue, 1, $newValue, 1));
if ($nameTagValue) {
$nameTagValue = json_encode(['value' => $nameTagValue]);
$this->db->Execute("\n INSERT INTO `governance`\n SET `env_id` = ?,\n `category` = ?,\n `name` = ?,\n `value` = ?,\n `enabled` = ?\n ON DUPLICATE KEY UPDATE\n `value` = ?,\n `enabled` = ?\n ", array($environment->id, $platform, 'aws.instance_name_format', $nameTagValue, 1, $nameTagValue, 1));
}
}
}
}
}
public function GetCloudUserData()
{
$dbFarmRole = $this->GetFarmRoleObject();
$baseurl = \Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host');
if ($this->isOpenstack() && $this->platform != SERVER_PLATFORMS::VERIZON) {
$platform = SERVER_PLATFORMS::OPENSTACK;
} else {
$platform = $this->platform;
}
$retval = array("farmid" => $this->farmId, "role" => implode(",", $dbFarmRole->GetRoleObject()->getBehaviors()), "httpproto" => \Scalr::config('scalr.endpoint.scheme'), "region" => $this->GetCloudLocation(), "hash" => $this->GetFarmObject()->Hash, "realrolename" => $dbFarmRole->GetRoleObject()->name, "szr_key" => $this->GetKey(), "serverid" => $this->serverId, 'p2p_producer_endpoint' => $baseurl . "/messaging", 'queryenv_url' => $baseurl . "/query-env", 'behaviors' => implode(",", $dbFarmRole->GetRoleObject()->getBehaviors()), 'farm_roleid' => $dbFarmRole->ID, 'roleid' => $dbFarmRole->RoleID, 'env_id' => $dbFarmRole->GetFarmObject()->EnvID, 'platform' => $platform, 'server_index' => $this->index, 'cloud_server_id' => $this->GetCloudServerID(), 'cloud_location_zone' => $this->cloudLocationZone, "owner_email" => $dbFarmRole->GetFarmObject()->createdByUserEmail);
$retval['message_format'] = 'json';
if (PlatformFactory::isOpenstack($this->platform) && $this->platform != SERVER_PLATFORMS::VERIZON) {
$retval["cloud_storage_path"] = "swift://";
} else {
switch ($this->platform) {
case SERVER_PLATFORMS::EC2:
$retval["s3bucket"] = $dbFarmRole->GetSetting(Entity\FarmRoleSetting::AWS_S3_BUCKET);
$retval["cloud_storage_path"] = "s3://" . $dbFarmRole->GetSetting(Entity\FarmRoleSetting::AWS_S3_BUCKET);
break;
case SERVER_PLATFORMS::RACKSPACE:
$retval["cloud_storage_path"] = "cf://scalr-{$this->GetFarmObject()->Hash}";
break;
case SERVER_PLATFORMS::GCE:
$retval["cloud_storage_path"] = "gcs://";
break;
}
}
// Custom settings
foreach ($dbFarmRole->GetSettingsByFilter("user-data") as $k => $v) {
$retval[str_replace("user-data", "custom", $k)] = $v;
}
return $retval;
}
/**
* @param string $platform
* @param string $cloudLocation
* @throws Exception
*/
public function xGetCloudServersListAction($platform, $cloudLocation)
{
if (!$this->environment->isPlatformEnabled($platform)) {
throw new Exception(sprintf('Cloud %s is not enabled for current environment', $platform));
}
$results = [];
$platformObj = PlatformFactory::NewPlatform($platform);
if ($platform == SERVER_PLATFORMS::GCE) {
$gce = $platformObj->getClient($this->environment);
$result = $gce->instances->listInstances($this->environment->keychain(SERVER_PLATFORMS::GCE)->properties[CloudCredentialsProperty::GCE_PROJECT_ID], $cloudLocation, []);
if (is_array($result->items)) {
foreach ($result->items as $server) {
if ($server->status != 'RUNNING') {
continue;
}
$ips = $platformObj->determineServerIps($gce, $server);
$itm = ['id' => $server->name, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $cloudLocation, 'isImporting' => false, 'isManaged' => false];
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(GCE_SERVER_PROPERTIES::SERVER_NAME, $server->name);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
} else {
if ($platform == SERVER_PLATFORMS::AZURE) {
// cloudLocation is resourceGroup
$t = $this->getEnvironment()->azure()->compute->virtualMachine->getList($this->getEnvironment()->keychain(SERVER_PLATFORMS::AZURE)->properties[CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $cloudLocation);
foreach ($t as $server) {
$itm = ['id' => $server->name, 'isImporting' => false, 'isManaged' => false];
$nicInfo = $server->properties->networkProfile->networkInterfaces[0]->id;
// get id and call
if (!empty($nicInfo->properties->ipConfigurations)) {
foreach ($nicInfo->properties->ipConfigurations as $ipConfig) {
$privateIp = $ipConfig->properties->privateIPAddress;
if ($ipConfig->properties->publicIPAddress) {
$publicIp = $ipConfig->properties->publicIPAddress->properties->ipAddress;
if ($publicIp) {
break;
}
}
}
}
$itm['localIp'] = $privateIp;
$itm['publicIp'] = $publicIp;
$itm['zone'] = $server->location;
$results[] = $itm;
}
} elseif (PlatformFactory::isOpenstack($platform)) {
$client = $this->environment->openstack($platform, $cloudLocation);
$r = $client->servers->list(true);
do {
foreach ($r as $server) {
if ($server->status != 'ACTIVE') {
continue;
}
$ips = $platformObj->determineServerIps($client, $server);
$itm = array('id' => $server->id, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $cloudLocation, 'isImporting' => false, 'isManaged' => false, 'fullInfo' => $server);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(OPENSTACK_SERVER_PROPERTIES::SERVER_ID, $server->id);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
} while (false !== ($r = $r->getNextPage()));
} elseif (PlatformFactory::isCloudstack($platform)) {
$client = $this->environment->cloudstack($platform);
$platformObj = PlatformFactory::NewPlatform($platform);
$r = $client->instance->describe(array('zoneid' => $cloudLocation));
if (count($r) > 0) {
foreach ($r as $server) {
$ips = $platformObj->determineServerIps($client, $server);
$itm = array('id' => $server->id, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $cloudLocation, 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID, $server->id);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
} elseif ($platform == SERVER_PLATFORMS::EC2) {
$client = $this->environment->aws($cloudLocation)->ec2;
$nextToken = null;
do {
if (isset($r)) {
$nextToken = $r->getNextToken();
}
$r = $client->instance->describe(null, null, $nextToken);
if (count($r)) {
foreach ($r as $reservation) {
/* @var $reservation Scalr\Service\Aws\Ec2\DataType\ReservationData */
foreach ($reservation->instancesSet as $instance) {
/* @var $instance Scalr\Service\Aws\Ec2\DataType\InstanceData */
if ($instance->instanceState->name != 'running') {
continue;
}
$itm = array('id' => $instance->instanceId, 'localIp' => $instance->privateIpAddress, 'publicIp' => $instance->ipAddress, 'zone' => $instance->placement->availabilityZone, 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(EC2_SERVER_PROPERTIES::INSTANCE_ID, $instance->instanceId);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
}
} while ($r->getNextToken());
}
}
$this->response->data(array('data' => $results));
}
/**
* Remove SSH keys
*
* @param JsonData $sshKeyId json array of sshKeyId to remove
* @throws Scalr_Exception_InsufficientPermissions
* @throws Scalr_UI_Exception_NotFound
*/
public function xRemoveAction(JsonData $sshKeyId)
{
$errors = [];
foreach ($sshKeyId as $id) {
try {
$sshKey = Scalr_SshKey::init()->loadById($id);
$this->user->getPermissions()->validate($sshKey);
if ($sshKey->type == Scalr_SshKey::TYPE_GLOBAL) {
if ($sshKey->platform == SERVER_PLATFORMS::EC2) {
$aws = $this->getEnvironment()->aws($sshKey->cloudLocation);
$aws->ec2->keyPair->delete($sshKey->cloudKeyName);
$sshKey->delete();
} elseif (PlatformFactory::isOpenstack($sshKey->platform)) {
$os = $this->getEnvironment()->openstack($sshKey->platform, $sshKey->cloudLocation);
$os->servers->keypairs->delete($sshKey->cloudKeyName);
$sshKey->delete();
} else {
$sshKey->delete();
}
} else {
//TODO:
}
} catch (Exception $e) {
$errors[] = $e->getMessage();
}
}
if (count($errors)) {
$this->response->warning("SSH key(s) successfully removed, but some errors occurred:\n" . implode("\n", $errors));
} else {
$this->response->success('SSH key(s) successfully removed');
}
}
public function getContext()
{
$data = array();
if ($this->user) {
$data['user'] = array('userId' => $this->user->getId(), 'clientId' => $this->user->getAccountId(), 'userName' => $this->user->getEmail(), 'gravatarHash' => $this->user->getGravatarHash(), 'envId' => $this->getEnvironment() ? $this->getEnvironmentId() : 0, 'envName' => $this->getEnvironment() ? $this->getEnvironment()->name : '', 'envVars' => $this->getEnvironment() ? $this->getEnvironment()->getPlatformConfigValue(Scalr_Environment::SETTING_UI_VARS) : '', 'type' => $this->user->getType());
$envVars = json_decode($data['user']['envVars'], true);
$betaMode = $envVars && $envVars['beta'] == 1;
if (!$this->user->isAdmin()) {
$data['farms'] = $this->db->getAll('SELECT id, name FROM farms WHERE env_id = ? ORDER BY name', array($this->getEnvironmentId()));
if ($this->user->getAccountId() != 0) {
$data['flags'] = $this->user->getAccount()->getFeaturesList();
$data['user']['userIsTrial'] = $this->user->getAccount()->getSetting(Scalr_Account::SETTING_IS_TRIAL) == '1' ? true : false;
} else {
$data['flags'] = array();
}
$data['flags']['billingExists'] = \Scalr::config('scalr.billing.enabled');
$data['flags']['featureUsersPermissions'] = $this->user->getAccount()->isFeatureEnabled(Scalr_Limits::FEATURE_USERS_PERMISSIONS);
$data['flags']['wikiUrl'] = \Scalr::config('scalr.ui.wiki_url');
$data['flags']['supportUrl'] = \Scalr::config('scalr.ui.support_url');
if ($data['flags']['supportUrl'] == '/core/support') {
$data['flags']['supportUrl'] .= '?X-Requested-Token=' . Scalr_Session::getInstance()->getToken();
}
$data['acl'] = $this->request->getAclRoles()->getAllowedArray(true);
if ($this->user->isAccountOwner()) {
if (!$this->user->getAccount()->getSetting(Scalr_Account::SETTING_DATE_ENV_CONFIGURED)) {
if (count($this->environment->getEnabledPlatforms()) == 0) {
$data['flags']['needEnvConfig'] = Scalr_Environment::init()->loadDefault($this->user->getAccountId())->id;
}
}
}
$data['environments'] = $this->user->getEnvironments();
if ($this->getEnvironment() && $this->user->isTeamOwner()) {
$data['user']['isTeamOwner'] = true;
}
}
$data['platforms'] = array();
$allowedClouds = (array) \Scalr::config('scalr.allowed_clouds');
foreach (SERVER_PLATFORMS::getList() as $platform => $platformName) {
if (!in_array($platform, $allowedClouds) || $platform == SERVER_PLATFORMS::UCLOUD && !$this->request->getHeaderVar('Interface-Beta')) {
continue;
}
$data['platforms'][$platform] = array('public' => PlatformFactory::isPublic($platform), 'enabled' => $this->user->isAdmin() ? true : !!$this->environment->isPlatformEnabled($platform), 'name' => $platformName);
if (!$this->user->isAdmin()) {
if ($platform == SERVER_PLATFORMS::EC2 && $this->environment->status == Scalr_Environment::STATUS_INACTIVE && $this->environment->getPlatformConfigValue('system.auto-disable-reason')) {
$data['platforms'][$platform]['config'] = array('autoDisabled' => true);
}
if (PlatformFactory::isOpenstack($platform) && $data['platforms'][$platform]['enabled']) {
$data['platforms'][$platform]['config'] = array(OpenstackPlatformModule::EXT_SECURITYGROUPS_ENABLED => PlatformFactory::NewPlatform($platform)->getConfigVariable(OpenstackPlatformModule::EXT_SECURITYGROUPS_ENABLED, $this->getEnvironment(), false), OpenstackPlatformModule::EXT_LBAAS_ENABLED => PlatformFactory::NewPlatform($platform)->getConfigVariable(OpenstackPlatformModule::EXT_LBAAS_ENABLED, $this->getEnvironment(), false), OpenstackPlatformModule::EXT_FLOATING_IPS_ENABLED => PlatformFactory::NewPlatform($platform)->getConfigVariable(OpenstackPlatformModule::EXT_FLOATING_IPS_ENABLED, $this->getEnvironment(), false), OpenstackPlatformModule::EXT_CINDER_ENABLED => PlatformFactory::NewPlatform($platform)->getConfigVariable(OpenstackPlatformModule::EXT_CINDER_ENABLED, $this->getEnvironment(), false), OpenstackPlatformModule::EXT_SWIFT_ENABLED => PlatformFactory::NewPlatform($platform)->getConfigVariable(OpenstackPlatformModule::EXT_SWIFT_ENABLED, $this->getEnvironment(), false));
}
}
}
$data['flags']['uiStorageTime'] = $this->user->getSetting(Scalr_Account_User::SETTING_UI_STORAGE_TIME);
$data['flags']['uiStorage'] = $this->user->getVar(Scalr_Account_User::VAR_UI_STORAGE);
$data['flags']['allowManageAnalytics'] = (bool) Scalr::isAllowedAnalyticsOnHostedScalrAccount($this->environment->clientId);
}
if ($this->user) {
$data['tags'] = Tag::getAll($this->user->getAccountId());
}
$data['flags']['authMode'] = $this->getContainer()->config->get('scalr.auth_mode');
$data['flags']['specialToken'] = Scalr_Session::getInstance()->getToken();
$data['flags']['hostedScalr'] = (bool) Scalr::isHostedScalr();
$data['flags']['analyticsEnabled'] = $this->getContainer()->analytics->enabled;
return $data;
}
/**
* Gets a normalized url for an each platform
*
* @param string $platform Cloud platform
* @return string Returns url
*/
public function getUrl($platform)
{
if (!isset($this->aUrl[$platform])) {
if ($platform == \SERVER_PLATFORMS::EC2) {
$value = '';
} else {
if (PlatformFactory::isOpenstack($platform)) {
$value = CloudLocation::normalizeUrl($this->env->getPlatformConfigValue($platform . '.' . OpenstackPlatformModule::KEYSTONE_URL));
} else {
if (PlatformFactory::isCloudstack($platform)) {
$value = CloudLocation::normalizeUrl($this->env->getPlatformConfigValue($platform . '.' . CloudstackPlatformModule::API_URL));
} else {
if ($platform == \SERVER_PLATFORMS::GCE) {
$value = '';
}
}
}
}
$this->aUrl[$platform] = $value;
}
return $this->aUrl[$platform];
}
/**
* xGetPlatformInstanceTypesAction
*
* @param string $platform The name of the cloud platform
* @param string $cloudLocation The cloud location
* @param string $envId optional The identifier of the environment
* @param string $effectiveDate optional The date on which prices should be applied YYYY-MM-DD
* @throws \Exception
*/
public function xGetPlatformInstanceTypesAction($platform, $cloudLocation, $envId = null, $effectiveDate = null)
{
list($curDate, $effectiveDate) = $this->handleEffectiveDate($effectiveDate);
$pm = PlatformFactory::NewPlatform($platform);
$env = null;
$url = '';
try {
if (!empty($envId)) {
$env = Scalr_Environment::init()->loadById($envId);
if (PlatformFactory::isOpenstack($platform)) {
$key = Entity\CloudCredentialsProperty::OPENSTACK_KEYSTONE_URL;
} else {
if (PlatformFactory::isCloudstack($platform)) {
$key = Entity\CloudCredentialsProperty::CLOUDSTACK_API_URL;
} else {
throw new Exception('This action is not yet supported for the specified cloud platform.');
}
}
if (empty($url)) {
$url = $env->keychain($platform)->properties[$key];
}
} else {
if ($platform == SERVER_PLATFORMS::EC2 || $platform == SERVER_PLATFORMS::GCE) {
$gcenvid = $this->getEnvIdByPlatform($platform);
$env = Scalr_Environment::init()->loadById($gcenvid);
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), 'not found')) {
//Tries to find url from the cloud_locations table
if (empty($url) && (PlatformFactory::isOpenstack($platform) || PlatformFactory::isCloudstack($platform))) {
$clEntity = CloudLocation::findOne([['platform' => $platform], ['cloudLocation' => $cloudLocation]], null, ['updated' => false]);
if ($clEntity instanceof CloudLocation) {
$url = $clEntity->url;
}
}
} else {
throw $e;
}
}
$result = $this->getTypesWithPrices($cloudLocation, $url, $pm, $platform, $effectiveDate, $env);
$this->response->data(['data' => $result]);
}
/**
* Gets a normalized url for an each platform
*
* @param string $platform Cloud platform
* @return string Returns url
*/
public function getUrl($platform)
{
if (!isset($this->aUrl[$platform])) {
if ($platform == \SERVER_PLATFORMS::EC2 || $platform == \SERVER_PLATFORMS::GCE || $platform == \SERVER_PLATFORMS::AZURE) {
$value = '';
} else {
if (PlatformFactory::isOpenstack($platform)) {
$value = CloudLocation::normalizeUrl($this->env->keychain($platform)->properties[CloudCredentialsProperty::OPENSTACK_KEYSTONE_URL]);
} else {
if (PlatformFactory::isCloudstack($platform)) {
$value = CloudLocation::normalizeUrl($this->env->keychain($platform)->properties[CloudCredentialsProperty::CLOUDSTACK_API_URL]);
}
}
}
$this->aUrl[$platform] = $value;
}
return $this->aUrl[$platform];
}
public function isOpenstack()
{
return PlatformFactory::isOpenstack($this->Platform);
}
public function run1($stage)
{
if ($this->hasTable('images')) {
$this->db->Execute('DROP TABLE images');
// drop old table if existed
}
$this->db->Execute("CREATE TABLE `images` (\n `hash` binary(16) NOT NULL,\n `id` varchar(128) NOT NULL DEFAULT '',\n `env_id` int(11) NULL DEFAULT NULL,\n `bundle_task_id` int(11) NULL DEFAULT NULL,\n `platform` varchar(25) NOT NULL DEFAULT '',\n `cloud_location` varchar(255) NOT NULL DEFAULT '',\n `os_family` varchar(25) NULL DEFAULT NULL,\n `os_version` varchar(10) NULL DEFAULT NULL,\n `os_name` varchar(255) NULL DEFAULT NULL,\n `created_by_id` int(11) NULL DEFAULT NULL,\n `created_by_email` varchar(100) NULL DEFAULT NULL,\n `architecture` enum('i386','x86_64') NOT NULL DEFAULT 'x86_64',\n `is_deprecated` tinyint(1) NOT NULL DEFAULT '0',\n `source` enum('BundleTask','Manual') NOT NULL DEFAULT 'Manual',\n `type` varchar(20) NULL DEFAULT NULL,\n `status` varchar(20) NOT NULL,\n `status_error` varchar(255) NULL DEFAULT NULL,\n `agent_version` varchar(20) NULL DEFAULT NULL,\n PRIMARY KEY (`hash`),\n UNIQUE KEY `idx_id` (`env_id`, `id`, `platform`, `cloud_location`),\n CONSTRAINT `fk_images_client_environmnets_id` FOREIGN KEY (`env_id`) REFERENCES `client_environments` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION\n ) ENGINE=InnoDB DEFAULT CHARSET=latin1;\n ");
$allRecords = 0;
$excludedCL = 0;
$excludedMissing = 0;
// convert
$tasks = [];
foreach ($this->db->GetAll('SELECT id as bundle_task_id, client_id as account_id, env_id, platform, snapshot_id as id, cloud_location, os_family, os_name,
os_version, created_by_id, created_by_email, bundle_type as type FROM bundle_tasks WHERE status = ?', [\SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS]) as $t) {
if (!is_array($tasks[$t['env_id']])) {
$tasks[$t['env_id']] = [];
}
$allRecords++;
$tasks[$t['env_id']][] = $t;
}
foreach ($this->db->GetAll('SELECT r.client_id as account_id, r.env_id, ri.platform, ri.image_id as id, ri.cloud_location, ri.os_family, ri.os_name,
ri.os_version, r.added_by_userid as created_by_id, r.added_by_email as created_by_email, ri.agent_version FROM role_images ri JOIN roles r ON r.id = ri.role_id') as $t) {
if (!is_array($tasks[$t['env_id']])) {
$tasks[$t['env_id']] = [];
}
$allRecords++;
$tasks[$t['env_id']][] = $t;
}
foreach ($tasks as $id => $e) {
if ($id == 0) {
continue;
}
try {
$env = (new \Scalr_Environment())->loadById($id);
} catch (\Exception $e) {
$this->console->warning('Invalid environment %d: %s', $id, $e->getMessage());
continue;
}
foreach ($e as $t) {
// check if snapshot exists
$add = false;
if ($this->db->GetOne('SELECT id FROM images WHERE id = ? AND env_id = ? AND platform = ? AND cloud_location = ? LIMIT 1', [$t['id'], $t['env_id'], $t['platform'], $t['cloud_location']])) {
continue;
}
if ($t['platform'] != \SERVER_PLATFORMS::GCE && !$t['cloud_location']) {
$excludedCL++;
continue;
}
try {
switch ($t['platform']) {
case \SERVER_PLATFORMS::EC2:
$snap = $env->aws($t['cloud_location'])->ec2->image->describe($t['id']);
if (count($snap)) {
$add = true;
$t['architecture'] = $snap->toArray()[0]['architecture'];
}
break;
case \SERVER_PLATFORMS::RACKSPACE:
$platform = PlatformFactory::NewPlatform(\SERVER_PLATFORMS::RACKSPACE);
/* @var $platform RackspacePlatformModule */
$client = \Scalr_Service_Cloud_Rackspace::newRackspaceCS($env->getPlatformConfigValue(RackspacePlatformModule::USERNAME, true, $t['cloud_location']), $env->getPlatformConfigValue(RackspacePlatformModule::API_KEY, true, $t['cloud_location']), $t['cloud_location']);
$snap = $client->getImageDetails($t['id']);
if ($snap) {
$add = true;
} else {
$excludedMissing++;
}
break;
case \SERVER_PLATFORMS::GCE:
$platform = PlatformFactory::NewPlatform(\SERVER_PLATFORMS::GCE);
/* @var $platform GoogleCEPlatformModule */
$client = $platform->getClient($env);
/* @var $client \Google_Service_Compute */
$projectId = $env->getPlatformConfigValue(GoogleCEPlatformModule::PROJECT_ID);
$snap = $client->images->get($projectId, str_replace($projectId . '/images/', '', $t['id']));
if ($snap) {
$add = true;
$t['architecture'] = 'x86_64';
} else {
$excludedMissing++;
}
break;
case \SERVER_PLATFORMS::EUCALYPTUS:
$snap = $env->eucalyptus($t['cloud_location'])->ec2->image->describe($t['id']);
if (count($snap)) {
$add = true;
$t['architecture'] = $snap->toArray()[0]['architecture'];
}
break;
default:
if (PlatformFactory::isOpenstack($t['platform'])) {
$snap = $env->openstack($t['platform'], $t['cloud_location'])->servers->getImage($t['id']);
if ($snap) {
$add = true;
$t['architecture'] = $snap->metadata->arch == 'x84-64' ? 'x84_64' : 'i386';
} else {
$excludedMissing++;
}
} else {
if (PlatformFactory::isCloudstack($t['platform'])) {
$snap = $env->cloudstack($t['platform'])->template->describe(['templatefilter' => 'executable', 'id' => $t['id'], 'zoneid' => $t['cloud_location']]);
if ($snap) {
if (isset($snap[0])) {
$add = true;
}
} else {
$excludedMissing++;
}
} else {
$this->console->warning('Unknown platform: %s', $t['platform']);
}
}
}
if ($add) {
$image = new Image();
$image->id = $t['id'];
$image->envId = $t['env_id'];
$image->bundleTaskId = $t['bundle_task_id'];
$image->platform = $t['platform'];
$image->cloudLocation = $t['cloud_location'];
$image->createdById = $t['created_by_id'];
$image->createdByEmail = $t['created_by_email'];
$image->architecture = $t['architecture'] ? $t['architecture'] : 'x86_64';
$image->isDeprecated = 0;
$image->source = $t['bundle_task_id'] ? 'BundleTask' : 'Manual';
$image->type = $t['type'];
$image->status = Image::STATUS_ACTIVE;
$image->agentVersion = $t['agent_version'];
$image->save();
} else {
$excludedMissing++;
}
} catch (\Exception $e) {
if (strpos($e->getMessage(), 'The resource could not be found') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'The requested URL / was not found on this server.') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'Not Found') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'was not found') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'Bad username or password') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'unable to verify user credentials and/or request signature') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'OpenStack error. Image not found.') !== FALSE) {
$excludedMissing++;
} else {
if (strpos($e->getMessage(), 'Neither api key nor password was provided for the OpenStack config.') !== FALSE) {
$excludedMissing++;
} else {
$this->console->warning('SnapshotId: %s, envId: %d, error: %s', $t['id'], $t['env_id'], $e->getMessage());
}
}
}
}
}
}
}
}
}
}
}
$this->console->notice('Found %d records', $allRecords);
$this->console->notice('Excluded %d images because of null cloud_location', $excludedCL);
$this->console->notice('Excluded %d missed images', $excludedMissing);
}
public function dashboardAction()
{
$this->request->defineParams(array('farmId' => array('type' => 'int'), 'farmRoleId' => array('type' => 'int'), 'type'));
$dbFarm = DBFarm::LoadByID($this->getParam('farmId'));
$this->user->getPermissions()->validate($dbFarm);
if ($this->getParam('farmRoleId')) {
$dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId'));
if ($dbFarmRole->FarmID != $dbFarm->ID) {
throw new Exception("Role not found");
}
} elseif ($this->getParam('type')) {
foreach ($dbFarm->GetFarmRoles() as $sDbFarmRole) {
if ($sDbFarmRole->GetRoleObject()->hasBehavior($this->getParam('type'))) {
$dbFarmRole = $sDbFarmRole;
break;
}
}
if (!$dbFarmRole) {
throw new Exception("Role not found");
}
} else {
throw new Scalr_UI_Exception_NotFound();
}
$data = array('farmRoleId' => $dbFarmRole->ID, 'farmId' => $dbFarmRole->FarmID);
$data['dbType'] = $dbFarmRole->GetRoleObject()->getDbMsrBehavior();
if (!$data['dbType']) {
$this->response->failure("Unknown db type");
return;
}
switch ($data['dbType']) {
case ROLE_BEHAVIORS::MYSQL2:
case ROLE_BEHAVIORS::PERCONA:
case ROLE_BEHAVIORS::MARIADB:
$szrApiNamespace = Scalr_Net_Scalarizr_Client::NAMESPACE_MYSQL;
break;
case ROLE_BEHAVIORS::REDIS:
$szrApiNamespace = Scalr_Net_Scalarizr_Client::NAMESPACE_REDIS;
$data['extras'] = array(array('name' => 'Processes', 'value' => $dbFarmRole->GetSetting(Scalr_Db_Msr_Redis::NUM_PROCESSES)), array('name' => 'Persistence type', 'value' => $dbFarmRole->GetSetting(Scalr_Db_Msr_Redis::PERSISTENCE_TYPE)));
break;
case ROLE_BEHAVIORS::POSTGRESQL:
$szrApiNamespace = Scalr_Net_Scalarizr_Client::NAMESPACE_POSTGRESQL;
break;
}
// Get PMA details for MySQL / Percona
if (in_array($data['dbType'], array(ROLE_BEHAVIORS::MYSQL, ROLE_BEHAVIORS::MYSQL2, ROLE_BEHAVIORS::PERCONA, ROLE_BEHAVIORS::MARIADB))) {
$data['pma'] = $this->getPmaDetails($dbFarmRole);
}
$behavior = Scalr_Role_Behavior::loadByName($data['dbType']);
$masterServer = $behavior->getMasterServer($dbFarmRole);
if ($masterServer) {
// Get Storage details
$data['storage'] = $this->getDbStorageStatus($masterServer, $data['dbType']);
}
// Get Access details and DNS endpoints
$data['accessDetails'] = $this->getDbAccessDetails($dbFarmRole);
$data['name'] = ROLE_BEHAVIORS::GetName($data['dbType']);
// Get data bundle info
$bundlesEnabled = $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BUNDLE_ENABLED);
$lastActionTime = $dbFarmRole->GetSetting(Scalr_Db_Msr::getConstant("DATA_BUNDLE_LAST_TS"));
$data['bundles'] = array('history' => $this->db->GetAll("SELECT *, UNIX_TIMESTAMP(date) as date FROM services_db_backups_history WHERE `farm_role_id` = ? AND `operation` = ? ORDER BY id ASC", array($dbFarmRole->ID, 'bundle')), 'inProgress' => array('status' => (int) $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BUNDLE_IS_RUNNING), 'serverId' => $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BUNDLE_SERVER_ID)), 'last' => $lastActionTime ? Scalr_Util_DateTime::convertTz((int) $lastActionTime, 'd M Y \\a\\t H:i:s') : 'Never');
foreach ($data['bundles']['history'] as &$h) {
$h['date'] = Scalr_Util_DateTime::convertTz((int) $h['date'], 'd M Y \\a\\t H:i:s');
}
if ($bundlesEnabled) {
$period = $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BUNDLE_EVERY);
if ($lastActionTime) {
$nextTime = $lastActionTime + $period * 3600;
}
$data['bundles']['next'] = !$nextTime || $nextTime < time() ? "Within 30 minutes" : Scalr_Util_DateTime::convertTz((int) $nextTime, 'd M Y \\a\\t H:i:s');
$data['bundles']['schedule'] = "Every {$period} hours";
} else {
$data['bundles']['next'] = " - ";
$data['bundles']['schedule'] = "Auto-snapshotting disabled";
}
// Get backups info
$lastActionTime = $dbFarmRole->GetSetting(Scalr_Db_Msr::getConstant("DATA_BACKUP_LAST_TS"));
$nextTime = false;
$backupsEnabled = $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BACKUP_ENABLED);
$data['backups'] = array('history' => $this->db->GetAll("SELECT *, UNIX_TIMESTAMP(date) as date FROM services_db_backups_history WHERE `farm_role_id` = ? AND `operation` = ? ORDER BY id ASC", array($dbFarmRole->ID, 'backup')), 'inProgress' => array('status' => (int) $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BACKUP_IS_RUNNING), 'serverId' => $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BACKUP_SERVER_ID)), 'last' => $lastActionTime ? Scalr_Util_DateTime::convertTz((int) $lastActionTime, 'd M Y \\a\\t H:i:s') : 'Never', 'supported' => !PlatformFactory::isCloudstack($dbFarmRole->Platform) && (!PlatformFactory::isOpenstack($dbFarmRole->Platform) || PlatformFactory::NewPlatform($dbFarmRole->Platform)->getConfigVariable(OpenstackPlatformModule::EXT_SWIFT_ENABLED, $this->getEnvironment(), false)));
foreach ($data['backups']['history'] as &$h) {
$h['date'] = Scalr_Util_DateTime::convertTz((int) $h['date'], 'd M Y \\a\\t H:i:s');
}
if ($backupsEnabled) {
$period = $dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_BACKUP_EVERY);
if ($lastActionTime) {
$nextTime = $lastActionTime + $period * 3600;
}
$data['backups']['next'] = !$nextTime || $nextTime < time() ? "Within 30 minutes" : Scalr_Util_DateTime::convertTz((int) $nextTime, 'd M Y \\a\\t H:i:s');
$data['backups']['schedule'] = "Every {$period} hours";
} else {
$data['backups']['next'] = " - ";
$data['backups']['schedule'] = "Auto-backups disabled";
}
/*
if ($dbFarmRole->GetSetting(Scalr_Db_Msr::DATA_STORAGE_ENGINE) == 'lvm') {
$data['noDataBundleForSlaves'] = ($dbFarmRole->GetSetting(Scalr_Role_DbMsrBehavior::ROLE_NO_DATA_BUNDLE_FOR_SLAVES)) ? true : false;
}
*/
$conf = $this->getContainer()->config->get('scalr.load_statistics.connections.plotter');
foreach ($dbFarmRole->GetServersByFilter(array('status' => array(SERVER_STATUS::INIT, SERVER_STATUS::RUNNING, SERVER_STATUS::PENDING))) as $dbServer) {
$isMaster = $dbServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) == 1;
$serverRole = $isMaster ? 'master' : 'slave';
$serverInfo = array('status' => $dbServer->status, 'remoteIp' => $dbServer->remoteIp, 'localIp' => $dbServer->localIp, 'serverId' => $dbServer->serverId, 'cloudServerId' => $dbServer->GetCloudServerID(), 'cloudLocation' => $dbServer->GetCloudLocation(), 'serverRole' => $serverRole, 'index' => $dbServer->index);
$serverInfo['monitoring'] = array('farmId' => $dbFarmRole->FarmID, 'farmRoleId' => $dbFarmRole->ID, 'index' => $dbServer->index, 'hash' => $dbFarm->Hash, 'hostUrl' => "{$conf['scheme']}://{$conf['host']}:{$conf['port']}");
if ($dbServer->platform == SERVER_PLATFORMS::EC2) {
$serverInfo['cloudLocation'] = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::AVAIL_ZONE);
}
if ($dbServer->status == SERVER_STATUS::RUNNING) {
try {
$rStatus = $dbServer->scalarizr->{$szrApiNamespace}->replicationStatus();
if ($data['dbType'] != ROLE_BEHAVIORS::REDIS) {
$rStatus = (array) $rStatus->{$serverRole};
$replication = $rStatus;
} else {
if ($isMaster) {
$rStatus = (array) $rStatus->masters;
foreach ($rStatus as $port => $status) {
$rStatus['status'] = $status;
if ($status != 'up') {
break;
}
}
} else {
$rStatus = (array) $rStatus->slaves;
foreach ($rStatus as $port => $status) {
$rStatus['status'] = $status->status;
if ($status->status != 'up') {
break;
}
}
}
$replication = $rStatus;
}
if (in_array($data['dbType'], array(ROLE_BEHAVIORS::MYSQL2, ROLE_BEHAVIORS::PERCONA, ROLE_BEHAVIORS::MARIADB))) {
if ($rStatus['status'] == 'up' && $replication['seconds_behind_master'] > 0) {
$status = 'lagging';
} else {
$status = $rStatus['status'];
}
} elseif ($data['dbType'] == ROLE_BEHAVIORS::REDIS) {
$status = $rStatus['status'];
} elseif ($data['dbType'] == ROLE_BEHAVIORS::POSTGRESQL) {
if ($rStatus['status'] == 'up' && $replication['Xlog_delay'] > 1000) {
$status = 'lagging';
} else {
$status = $rStatus['status'];
}
}
$serverInfo['replication'] = array('status' => $status, $data['dbType'] => $replication);
} catch (Exception $e) {
$serverInfo['replication'] = array('status' => 'error', 'message' => $e->getMessage());
}
}
$data['servers'][] = $serverInfo;
}
$this->response->page('ui/db/manager/dashboard.js', $data, array('ui/monitoring/window.js'), array('ui/db/manager/dashboard.css'));
}
public function FarmAddRole($Alias, $FarmID, $RoleID, $Platform, $CloudLocation, array $Configuration = array())
{
try {
$dbFarm = DBFarm::LoadByID($FarmID);
if ($dbFarm->EnvID != $this->Environment->id) {
throw new Exception("N");
}
} catch (Exception $e) {
throw new Exception(sprintf("Farm #%s not found", $FarmID));
}
$this->user->getPermissions()->validate($dbFarm);
$this->restrictFarmAccess($dbFarm, Acl::PERM_FARMS_MANAGE);
$dbFarm->isLocked(true);
$governance = new Scalr_Governance($this->Environment->id);
$dbRole = DBRole::loadById($RoleID);
if ($dbRole->envId != 0) {
$this->user->getPermissions()->validate($dbRole);
}
foreach ($dbRole->getBehaviors() as $behavior) {
if ($behavior != ROLE_BEHAVIORS::BASE && $behavior != ROLE_BEHAVIORS::CHEF) {
throw new Exception("Only base roles supported to be added to farm via API");
}
}
$config = array('scaling.enabled' => 0, 'scaling.min_instances' => 1, 'scaling.max_instances' => 1, 'scaling.polling_interval' => 2, 'system.timeouts.launch' => 9600, 'system.timeouts.reboot' => 9600);
if (PlatformFactory::isOpenstack($Platform)) {
//TODO:
}
if ($Platform == SERVER_PLATFORMS::EC2) {
$config['aws.security_groups.list'] = json_encode(array('default', \Scalr::config('scalr.aws.security_group_name')));
$vpcId = $dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID);
if ($vpcId) {
if (!$Configuration['aws.vpc_subnet_id']) {
throw new Exception("Farm configured to run inside VPC. 'aws.vpc_subnet_id' is required");
}
$vpcRegion = $dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_REGION);
if ($CloudLocation != $vpcRegion) {
throw new Exception(sprintf("Farm configured to run inside VPC in %s region. Only roles in this region are allowed.", $vpcRegion));
}
$vpcGovernance = $governance->getValue('ec2', 'aws.vpc');
$vpcGovernanceIds = $governance->getValue('ec2', 'aws.vpc', 'ids');
$subnets = json_decode($Configuration['aws.vpc_subnet_id'], true);
if (count($subnets) == 0) {
throw new Exception("Subnets list is empty or json is incorrect");
}
$type = false;
foreach ($subnets as $subnetId) {
$platform = PlatformFactory::NewPlatform(SERVER_PLATFORMS::EC2);
$info = $platform->listSubnets($this->Environment, $CloudLocation, $vpcId, true, $subnetId);
if (substr($info['availability_zone'], 0, -1) != $vpcRegion) {
throw new Exception(sprintf("Only subnets from %s region are allowed according to VPC settings", $vpcRegion));
}
if ($vpcGovernance == 1) {
// Check valid subnets
if ($vpcGovernanceIds[$vpcId] && is_array($vpcGovernanceIds[$vpcId]) && !in_array($subnetId, $vpcGovernanceIds[$vpcId])) {
throw new Exception(sprintf("Only %s subnet(s) allowed by governance settings", implode(', ', $vpcGovernanceIds[$vpcId])));
}
// Check if subnets types
if ($vpcGovernanceIds[$vpcId] == "outbound-only") {
if ($info['type'] != 'private') {
throw new Exception("Only private subnets allowed by governance settings");
}
}
if ($vpcGovernanceIds[$vpcId] == "full") {
if ($info['type'] != 'public') {
throw new Exception("Only public subnets allowed by governance settings");
}
}
}
if (!$type) {
$type = $info['type'];
} else {
if ($type != $info['type']) {
throw new Exception("Mix of public and private subnets are not allowed. Please specify only public or only private subnets.");
}
}
}
}
}
if (PlatformFactory::isCloudstack($Platform)) {
$config['cloudstack.security_groups.list'] = json_encode(array('default', \Scalr::config('scalr.aws.security_group_name')));
}
if ($Platform == SERVER_PLATFORMS::GCE) {
$config['gce.network'] = 'default';
$config['gce.on-host-maintenance'] = 'MIGRATE';
}
if ($Configuration[Scalr_Role_Behavior_Chef::ROLE_CHEF_BOOTSTRAP] == 1 && !$Configuration[Scalr_Role_Behavior_Chef::ROLE_CHEF_ENVIRONMENT]) {
$config[Scalr_Role_Behavior_Chef::ROLE_CHEF_ENVIRONMENT] = '_default';
}
$config = array_merge($config, $Configuration);
$this->validateFarmRoleConfiguration($config);
if ($Platform == SERVER_PLATFORMS::GCE) {
$config['gce.cloud-location'] = $CloudLocation;
$config['gce.region'] = substr($CloudLocation, 0, -1);
}
$Alias = $this->stripValue($Alias);
if (strlen($Alias) < 4) {
throw new Exception("Role Alias should be longer than 4 characters");
}
if (!preg_match("/^[A-Za-z0-9]+[A-Za-z0-9-]*[A-Za-z0-9]+\$/si", $Alias)) {
throw new Exception("Alias should start and end with letter or number and contain only letters, numbers and dashes.");
}
if (!$this->Environment->isPlatformEnabled($Platform)) {
throw new Exception("'{$Platform}' cloud is not configured in your environment");
}
$images = $dbRole->__getNewRoleObject()->fetchImagesArray();
$locations = isset($images[$Platform]) ? array_keys($images[$Platform]) : [];
if (!in_array($CloudLocation, $locations) && $Platform != SERVER_PLATFORMS::GCE) {
throw new Exception(sprintf("Role '%s' doesn't have an image configured for cloud location '%s'", $dbRole->name, $CloudLocation));
}
if ($Alias) {
foreach ($dbFarm->GetFarmRoles() as $farmRole) {
if ($farmRole->Alias == $Alias) {
throw new Exception("Selected alias is already used by another role in selected farm");
}
}
}
$dbFarmRole = $dbFarm->AddRole($dbRole, $Platform, $CloudLocation, 1);
$dbFarmRole->Alias = $Alias ? $Alias : $dbRole->name;
foreach ($config as $k => $v) {
$dbFarmRole->SetSetting($k, trim($v), DBFarmRole::TYPE_CFG);
}
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->onFarmSave($dbFarm, $dbFarmRole);
}
$dbFarmRole->Save();
$response = $this->CreateInitialResponse();
$response->FarmRoleID = $dbFarmRole->ID;
return $response;
}
private function getPlatformService($platform, $cloudLocation)
{
if ($platform == SERVER_PLATFORMS::EC2 || $platform == SERVER_PLATFORMS::EUCALYPTUS) {
$method = $platform == SERVER_PLATFORMS::EC2 ? 'aws' : 'eucalyptus';
return $this->getEnvironment()->{$method}($cloudLocation)->ec2->securityGroup;
} elseif (PlatformFactory::isOpenstack($platform)) {
$openstack = $this->getEnvironment()->openstack($platform, $cloudLocation);
return $openstack;
} elseif (PlatformFactory::isCloudstack($platform)) {
return $this->getEnvironment()->cloudstack($platform)->securityGroup;
} else {
throw new Exception('Platform is not supported');
}
}
public function _provider($from, $to, $action)
{
switch ($action) {
case static::ACT_CONVERT_TO_OBJECT:
/* @var $from Entity\CloudCredentials */
if (PlatformFactory::isOpenstack($from->cloud, true) || PlatformFactory::isCloudstack($from->cloud)) {
$to->provider = $from->cloud;
}
break;
case static::ACT_CONVERT_TO_ENTITY:
/* @var $to Entity\CloudCredentials */
break;
case static::ACT_GET_FILTER_CRITERIA:
if (!(PlatformFactory::isOpenstack($from->provider, true) || PlatformFactory::isCloudstack($from->provider))) {
throw new ApiErrorException(400, ErrorMessage::ERR_INVALID_VALUE, "Unknown cloud provider");
}
return [['cloud' => $from->provider]];
}
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
// Collect information from database
$servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/** @var DBServer $DBServer */
try {
if ($DBServer->cloudLocation) {
try {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})"));
$p = PlatformFactory::NewPlatform($DBServer->platform);
$list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED"));
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
if (!$serverInfo) {
if ($p->debugLog) {
Logger::getLogger('Openstack')->fatal($p->debugLog);
}
if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
if (PlatformFactory::isOpenstack($DBServer->platform)) {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform)));
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId));
}
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0");
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
/*
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped')
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
} catch (Exception $e) {}
*/
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform)) {
if ($DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
// if server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} else {
// If server was suspended during initialization - we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
//TODO: Depends on resume strategy need to set server status
// For Openstack we need to re-accociate IPs
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(10);
print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
/**
* Remove SSH keys
*
* @param JsonData $sshKeyId json array of sshKeyId to remove
*/
public function xRemoveAction(JsonData $sshKeyId)
{
$this->request->restrictAccess(Acl::RESOURCE_SECURITY_SSH_KEYS, Acl::PERM_SECURITY_SSH_KEYS_MANAGE);
$errors = [];
$processed = [];
foreach ($sshKeyId as $id) {
try {
/* @var $sshKey SshKey */
$sshKey = SshKey::findPk($id);
if ($sshKey) {
$this->checkPermissions($sshKey, true);
if ($sshKey->type == SshKey::TYPE_GLOBAL) {
if ($sshKey->platform == SERVER_PLATFORMS::EC2) {
$aws = $this->getEnvironment()->aws($sshKey->cloudLocation);
$aws->ec2->keyPair->delete($sshKey->cloudKeyName);
$sshKey->delete();
} elseif (PlatformFactory::isOpenstack($sshKey->platform)) {
$os = $this->getEnvironment()->openstack($sshKey->platform, $sshKey->cloudLocation);
$os->servers->keypairs->delete($sshKey->cloudKeyName);
$sshKey->delete();
} else {
$sshKey->delete();
}
$processed[] = $sshKey->id;
} else {
//TODO:
}
} else {
$errors[] = sprintf('SshKey [%d] was not found', $id);
}
} catch (\Scalr\Service\OpenStack\Exception\OpenStackException $e) {
if (strstr($e->getMessage(), 'not found') || strstr($e->getMessage(), 'not be found')) {
$sshKey->delete();
$processed[] = $sshKey->id;
} else {
$errors[] = $e->getMessage();
}
} catch (Exception $e) {
$errors[] = $e->getMessage();
}
}
$this->response->data(['processed' => $processed]);
if (count($errors)) {
$this->response->warning((count($processed) ? "SSH key(s) successfully removed, but some errors occurred:\n" : '') . implode("\n", $errors));
} else {
$this->response->success('SSH key(s) successfully removed');
}
}
public function FarmAddRole($Alias, $FarmID, $RoleID, $Platform, $CloudLocation, array $Configuration = array())
{
$this->restrictAccess(Acl::RESOURCE_FARMS, Acl::PERM_FARMS_MANAGE);
try {
$dbFarm = DBFarm::LoadByID($FarmID);
if ($dbFarm->EnvID != $this->Environment->id) {
throw new Exception("N");
}
} catch (Exception $e) {
throw new Exception(sprintf("Farm #%s not found", $FarmID));
}
$this->user->getPermissions()->validate($dbFarm);
$dbFarm->isLocked(true);
$dbRole = DBRole::loadById($RoleID);
if ($dbRole->envId != 0) {
$this->user->getPermissions()->validate($dbRole);
}
foreach ($dbRole->getBehaviors() as $behavior) {
if ($behavior != ROLE_BEHAVIORS::BASE && $behavior != ROLE_BEHAVIORS::CHEF) {
throw new Exception("Only base roles supported to be added to farm via API");
}
}
$config = array('scaling.enabled' => 0, 'scaling.min_instances' => 1, 'scaling.max_instances' => 1, 'scaling.polling_interval' => 2, 'system.timeouts.launch' => 9600, 'system.timeouts.reboot' => 9600);
if (PlatformFactory::isOpenstack($Platform)) {
//TODO:
}
if ($Platform == SERVER_PLATFORMS::EC2) {
$config['aws.security_groups.list'] = json_encode(array('default', \Scalr::config('scalr.aws.security_group_name')));
}
if ($Configuration[Scalr_Role_Behavior_Chef::ROLE_CHEF_BOOTSTRAP] == 1 && !$Configuration[Scalr_Role_Behavior_Chef::ROLE_CHEF_ENVIRONMENT]) {
$config[Scalr_Role_Behavior_Chef::ROLE_CHEF_ENVIRONMENT] = '_default';
}
$config = array_merge($config, $Configuration);
$this->validateFarmRoleConfiguration($config);
$Alias = $this->stripValue($Alias);
if (strlen($Alias) < 4) {
throw new Exception("Role Alias should be longer than 4 characters");
}
if (preg_match("/^[^A-Za-z0-9_-]+\$/", $Alias)) {
throw new Exception("Role Alias should has only 'A-Za-z0-9-_' characters");
}
if (!PlatformFactory::isOpenstack($Platform) && $Platform != SERVER_PLATFORMS::EC2) {
throw new Exception("Only EC2 and Openstack roles are supported by this method");
}
if (!$this->Environment->isPlatformEnabled($Platform)) {
throw new Exception("'{$Platform}' cloud is not configured in your environment");
}
$locations = $dbRole->getCloudLocations($Platform);
if (!in_array($CloudLocation, $locations)) {
throw new Exception(sprintf("Role '%s' doesn't have image configured for cloud location '%s'", $dbRole->name, $CloudLocation));
}
if ($Alias) {
foreach ($dbFarm->GetFarmRoles() as $farmRole) {
if ($farmRole->Alias == $Alias) {
throw new Exception("Selected alias is already used by another role in selected farm");
}
}
}
$dbFarmRole = $dbFarm->AddRole($dbRole, $Platform, $CloudLocation, 1);
$dbFarmRole->Alias = $Alias ? $Alias : $dbRole->name;
foreach ($config as $k => $v) {
$dbFarmRole->SetSetting($k, $v, DBFarmRole::TYPE_CFG);
}
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->onFarmSave($dbFarm, $dbFarmRole);
}
$dbFarmRole->Save();
$response = $this->CreateInitialResponse();
$response->FarmRoleID = $dbFarmRole->ID;
return $response;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//The list of the suspension information about cloud platforms
$this->aSuspensionInfo = [];
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform);
$jobStartTime = microtime(true);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers\n WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm and platform
return;
}
$this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform);
$config = \Scalr::getContainer()->config;
/*
if ($request->platform) {
$p = PlatformFactory::NewPlatform($request->platform);
$p->ClearCache();
}
*/
$p = PlatformFactory::NewPlatform($request->platform);
foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) {
/* @var $DBServer \DBServer */
//Get platform suspension info
$suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId);
//If the cloud platform is suspended we should not process it
if ($suspensionInfo->isSuspended()) {
continue;
}
try {
//1. We need to check that server is exists in cloud and not missed.
// (On Openstack server can be missed and should not be terminated)
$cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation);
if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
//For Azure we need to pass resource group instead of cloudLocation
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP));
} else {
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
}
//We successfully polled cloud so can resume suspension status for the cloud platform
if ($suspensionInfo->isPendingSuspend()) {
$suspensionInfo->resume();
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
continue;
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId));
continue;
}
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]);
} else {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
// If openstack server is in ERROR state we need more details
if ($openstackErrorState) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$status = empty($info['Status']) ? false : $info['Status'];
} catch (Exception $e) {
}
}
if (empty($status)) {
$status = $DBServer->GetRealStatus()->getName();
}
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
if ($DBServer->status != \SERVER_STATUS::RESUMING) {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} else {
// Need to clear cache, because this situation happens only when cache is stale.
$p->ClearCache();
}
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
$update = [];
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$update['status'] = \SERVER_STATUS::RESUMING;
$update['dateAdded'] = date("Y-m-d H:i:s");
$DBServer->update($update);
unset($update);
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) {
$elasticIpAssigned = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
$elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($elasticIpAddress) {
$ipaddresses['remoteIp'] = $elasticIpAddress;
$DBServer->remoteIp = $elasticIpAddress;
$elasticIpAssigned = true;
}
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) {
$DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]);
}
//Add tags
Ec2Helper::createObjectTags($DBServer);
}
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
// Check scalarizr deployment status
$env = $DBServer->GetEnvironmentObject();
$azure = $env->azure();
$info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME));
$extensions = !empty($info->extensions) ? $info->extensions : [];
foreach ($extensions as $extension) {
/* @var $extension ExtensionData */
if ($extension->name == 'scalarizr') {
$extStatus = $extension->statuses[0];
/* @var $extStatus StatusData */
if ($extStatus->level == 'Error') {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]);
}
}
}
}
} else {
AzureHelper::setupScalrAgent($DBServer);
}
}
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
\Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scriptingEvent = false;
$eventName = null;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$eventName = "hostInit";
$scriptingEvent = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$eventName = "hostUp";
$scriptingEvent = EVENT_TYPE::HOST_UP;
}
if ($scriptingEvent && $dtadded) {
$hasPendingMessages = !!$db->GetOne("\n SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n ", [$DBServer->serverId]);
$scriptingTimeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scriptingEvent, $DBServer->farmRoleId]);
if ($scriptingTimeout) {
$launchTimeout = $launchTimeout + $scriptingTimeout;
}
if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting() && !$elasticIpAssigned) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
//TODO: Terminated outside scalr while in SUSPENDED state
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
// Private IP cannot be removed (only changed).
if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null;
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() > $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
$this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime);
return $request;
}
/**
* @param string $platform
* @param string $cloudLocation
* @throws Exception
*/
public function xGetInstanceTypesAction($platform, $cloudLocation = null)
{
if (!in_array($platform, $this->getEnvironment()->getEnabledPlatforms())) {
throw new Exception(sprintf('Platform "%s" is not enabled', $platform));
}
$p = PlatformFactory::NewPlatform($platform);
if (PlatformFactory::isOpenstack($platform) && !$cloudLocation) {
$locations = $p->getLocations($this->getEnvironment());
if (empty($locations)) {
throw new Exception(sprintf("Unable to retrieve the list of cloud locations for platform %s; " . "the cloud API may be down or unreachable, or the credentials provided to Scalr are invalid.", $platform));
}
$keys = array_keys($locations);
$cloudLocation = array_pop($keys);
}
$data = [];
foreach ($p->getInstanceTypes($this->getEnvironment(), $cloudLocation, true) as $id => $value) {
$data[] = array_merge(['id' => (string) $id], $value);
}
$this->response->data(array('data' => $data));
}
public function xGetCloudServersListAction()
{
$this->request->defineParams(array('platform', 'cloudLocation'));
if (!$this->environment->isPlatformEnabled($this->getParam('platform'))) {
throw new Exception(sprintf('Cloud %s is not enabled for current environment', $this->getParam('platform')));
}
$results = array();
$platform = PlatformFactory::NewPlatform($this->getParam('platform'));
//TODO: Added support for GCE
if ($this->getParam('platform') == SERVER_PLATFORMS::GCE) {
$gce = $platform->getClient($this->environment, $this->getParam('cloudLocation'));
$result = $gce->instances->listInstances($this->environment->getPlatformConfigValue(GoogleCEPlatformModule::PROJECT_ID), $this->getParam('cloudLocation'), array());
if (is_array($result->items)) {
foreach ($result->items as $server) {
if ($server->status != 'RUNNING') {
continue;
}
$ips = $platform->determineServerIps($gce, $server);
$itm = array('id' => $server->name, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $this->getParam('cloudLocation'), 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(GCE_SERVER_PROPERTIES::SERVER_NAME, $server->name);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
} elseif (PlatformFactory::isOpenstack($this->getParam('platform'))) {
$client = $this->environment->openstack($this->getParam('platform'), $this->getParam('cloudLocation'));
$r = $client->servers->list(true);
do {
foreach ($r as $server) {
if ($server->status != 'ACTIVE') {
continue;
}
$ips = $platform->determineServerIps($client, $server);
$itm = array('id' => $server->id, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $this->getParam('cloudLocation'), 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(OPENSTACK_SERVER_PROPERTIES::SERVER_ID, $server->id);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
} while (false !== ($r = $r->getNextPage()));
} elseif (PlatformFactory::isCloudstack($this->getParam('platform'))) {
$client = $this->environment->cloudstack($this->getParam('platform'));
$platform = PlatformFactory::NewPlatform($this->getParam('platform'));
$r = $client->instance->describe(array('zoneid' => $this->getParam('cloudLocation')));
if (count($r) > 0) {
foreach ($r as $server) {
$ips = $platform->determineServerIps($client, $server);
$itm = array('id' => $server->id, 'localIp' => $ips['localIp'], 'publicIp' => $ips['remoteIp'], 'zone' => $this->getParam('cloudLocation'), 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID, $server->id);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
} elseif ($this->getParam('platform') == SERVER_PLATFORMS::EC2) {
$client = $this->environment->aws($this->getParam('cloudLocation'))->ec2;
$nextToken = null;
do {
if (isset($r)) {
$nextToken = $r->getNextToken();
}
$r = $client->instance->describe(null, null, $nextToken);
if (count($r)) {
foreach ($r as $reservation) {
/* @var $reservation Scalr\Service\Aws\Ec2\DataType\ReservationData */
foreach ($reservation->instancesSet as $instance) {
/* @var $instance Scalr\Service\Aws\Ec2\DataType\InstanceData */
if ($instance->instanceState->name != 'running') {
continue;
}
$itm = array('id' => $instance->instanceId, 'localIp' => $instance->privateIpAddress, 'publicIp' => $instance->ipAddress, 'zone' => $instance->placement->availabilityZone, 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(EC2_SERVER_PROPERTIES::INSTANCE_ID, $instance->instanceId);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
}
} while ($r->getNextToken());
} elseif ($this->getParam('platform') == SERVER_PLATFORMS::EUCALYPTUS) {
$client = $this->environment->eucalyptus($this->getParam('cloudLocation'))->ec2;
$r = $client->instance->describe(null, null, $nextToken);
if (count($r)) {
foreach ($r as $reservation) {
/* @var $reservation Scalr\Service\Aws\Ec2\DataType\ReservationData */
foreach ($reservation->instancesSet as $instance) {
/* @var $instance Scalr\Service\Aws\Ec2\DataType\InstanceData */
if ($instance->instanceState->name != 'running') {
continue;
}
$itm = array('id' => $instance->instanceId, 'localIp' => $instance->privateIpAddress, 'publicIp' => $instance->ipAddress, 'zone' => $instance->placement->availabilityZone, 'isImporting' => false, 'isManaged' => false);
//Check is instance already importing
try {
$dbServer = DBServer::LoadByPropertyValue(EUCA_SERVER_PROPERTIES::INSTANCE_ID, $instance->instanceId);
if ($dbServer && $dbServer->status != SERVER_STATUS::TERMINATED) {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$itm['isImporting'] = true;
} else {
$itm['isManaged'] = true;
}
$itm['serverId'] = $dbServer->serverId;
}
} catch (Exception $e) {
}
$results[] = $itm;
}
}
}
}
$this->response->data(array('data' => $results));
}
/**
* xGetPlatformInstanceTypesAction
*
* @param string $platform The name of the cloud platform
* @param string $cloudLocation The cloud location
* @param string $envId optional The identifier of the environment
* @param string $effectiveDate optional The date on which prices should be applied YYYY-MM-DD
* @throws \Exception
*/
public function xGetPlatformInstanceTypesAction($platform, $cloudLocation, $envId = null, $effectiveDate = null)
{
list($curDate, $effectiveDate) = $this->handleEffectiveDate($effectiveDate);
$pm = PlatformFactory::NewPlatform($platform);
$env = null;
$url = '';
try {
if (!empty($envId)) {
$env = Scalr_Environment::init()->loadById($envId);
if (PlatformFactory::isOpenstack($platform)) {
$key = $platform . '.' . OpenstackPlatformModule::KEYSTONE_URL;
} else {
if (PlatformFactory::isCloudstack($platform)) {
$key = $platform . '.' . CloudstackPlatformModule::API_URL;
} else {
if ($platform == SERVER_PLATFORMS::EUCALYPTUS) {
$key = EucalyptusPlatformModule::EC2_URL;
$url = $this->getContainer()->analytics->prices->normalizeUrl($env->getPlatformConfigValue($key, false, $cloudLocation));
} else {
throw new Exception('This action is not yet supported for the specified cloud platform.');
}
}
}
if (empty($url)) {
$url = $this->getContainer()->analytics->prices->normalizeUrl($env->getPlatformConfigValue($key));
}
} else {
if ($platform == SERVER_PLATFORMS::EC2 || $platform == SERVER_PLATFORMS::GCE) {
$gcenvid = $this->getPlatformEnvId($platform);
$env = Scalr_Environment::init()->loadById($gcenvid);
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), 'not found')) {
//Tries to find url from the cloud_locations table
if (empty($url) && (PlatformFactory::isOpenstack($platform) || PlatformFactory::isCloudstack($platform))) {
$clEntity = CloudLocation::findOne([['platform' => $platform], ['cloudLocation' => $cloudLocation]], ['updated' => false]);
if ($clEntity instanceof CloudLocation) {
$url = $clEntity->url;
}
}
} else {
throw $e;
}
}
$result = $this->getTypesWithPrices($cloudLocation, $url, $pm, $platform, $effectiveDate, $env);
$this->response->data(['data' => $result]);
}
public function xSaveCloudParamsAction()
{
$platform = $this->getParam('platform');
if (PlatformFactory::isCloudstack($platform)) {
$method = SERVER_PLATFORMS::CLOUDSTACK;
} elseif (PlatformFactory::isOpenstack($platform)) {
$method = SERVER_PLATFORMS::OPENSTACK;
} else {
$method = $platform;
}
$method = 'save' . ucfirst($method);
if (method_exists($this, $method)) {
$this->{$method}();
$suspensionInfo = new CloudPlatformSuspensionInfo($this->env->id, $platform);
$suspensionInfo->resume();
$this->response->data(array('params' => $this->getCloudParams($platform)));
} else {
$this->response->failure('Under construction ...');
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (!isset($request->farmRoleId)) {
//This is the farm with synchronous launch of roles
try {
$DBFarm = DBFarm::LoadByID($request->farmId);
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID);
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId);
throw $e;
}
//Gets the list of the roles
$list = $DBFarm->GetFarmRoles();
} else {
//This is asynchronous lauhch
try {
$DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId);
if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) {
//We don't need to handle inactive farms
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId);
throw $e;
}
$list = [$DBFarmRole];
}
$this->getLogger()->debug("Processing %s FarmRoles", count($list));
foreach ($list as $DBFarmRole) {
// Set Last polling time
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$disabledScaling = false;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') {
if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
// For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running)
// When 1 instance is already running, the rest is fully manual
$roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount();
if ($roleTotalInstances != 0) {
$disabledScaling = true;
}
} else {
if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$disabledScaling = true;
}
}
if ($disabledScaling) {
$this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias);
continue;
}
}
$farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name;
// Get current count of running and pending instances.
$this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
//Replacing the logger
$scalingManager->logger = $this->getLogger();
$scalingDecision = $scalingManager->makeScalingDecision();
$scalingDecisionDetails = $scalingManager->decisonInfo;
$this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails));
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue;
} else {
if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instances count after scaling
resolution the spare instances are running for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null));
continue;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
$ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR);
$useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN);
$isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ);
// Select instance that will be terminated
//
// Instances ordered by uptime (oldest wil be choosen)
// Instance cannot be mysql master
// Choose the one that was rebundled recently
$DBServer = null;
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($isRabbitMQ) {
$serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]);
if ($DBServer->index == 1 && $serverExists) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
$got_valid_instance = true;
}
//Check safe shutdown
if ($useSafeShutdown == 1) {
try {
$res = $DBServer->scalarizr->system->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
if ($res != '1') {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
$got_valid_instance = false;
}
}
}
// end while
if ($DBServer !== null && $got_valid_instance) {
$this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$ignoreFullHour) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
try {
if ($terminateStrategy == 'terminate') {
$DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} else {
$DBServer->suspend('SCALING_DOWN', false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
} catch (Exception $e) {
$this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId));
}
}
} else {
$this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID));
}
//break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution 'need more instances' for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias));
continue;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName)));
continue;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it.");
return;
}
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
$suspendedServer = null;
if ($terminateStrategy == 'suspend') {
$suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED));
if (count($suspendedServers) > 0) {
$suspendedServer = array_shift($suspendedServers);
}
}
if ($terminateStrategy == 'suspend' && $suspendedServer) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId));
}
if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
} else {
$platform = PlatformFactory::NewPlatform($suspendedServer->platform);
$platform->ResumeServer($suspendedServer);
}
}
}
}
return true;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm
return;
}
$this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID);
$config = \Scalr::getContainer()->config;
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/* @var $DBServer \DBServer */
try {
if ($DBServer->cloudLocation) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId);
$p = PlatformFactory::NewPlatform($DBServer->platform);
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId));
continue;
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag') {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
//More debug
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache)));
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}"));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
/* @var $env \Scalr_Environment */
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
//Saving the reason why this environment is disabled
$env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]);
return;
} elseif (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
$this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
if ($openstackErrorState) {
try {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId));
} catch (Exception $e) {
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer);
$DBServer->Save();
}
}
if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) {
$DBServer->status = \SERVER_STATUS::PENDING;
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1);
$DBServer->dateAdded = date("Y-m-d H:i:s");
$DBServer->Save();
} else {
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
\Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
}
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) {
$DBServer->remoteIp = $ipaddresses['remoteIp'];
$DBServer->localIp = $ipaddresses['localIp'];
$DBServer->Save();
}
//Add tags
Ec2Helper::createServerTags($DBServer);
}
if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scripting_event, $DBServer->farmRoleId]);
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
} elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) {
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
return $request;
}
