/**
* Creates clone for the farm
*
* @param string $name The name of the farm
* @param Scalr_Account_User $user The user object
* @param int $envId The identifier of the environment
* @return DBFarm Returns clone
*/
public function cloneFarm($name = false, Scalr_Account_User $user, $envId)
{
$account = $user->getAccount();
$account->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
$definition = $this->getDefinition();
if (!$name) {
$template = "";
if (!stristr($definition->name, "clone")) {
$template = $definition->name . ' (clone #%s)';
$i = 1;
} else {
preg_match("/^(.*?)\\(clone \\#([0-9]*)\\)\$/si", $definition->name, $matches);
$template = trim($matches[1]) . " (clone #%s)";
$i = $matches[2] + 1;
}
while (true) {
$name = sprintf($template, $i);
if (!$this->DB->GetOne("SELECT id FROM farms WHERE name = ? AND env_id = ? LIMIT 1", array($name, $this->EnvID))) {
break;
} else {
$i++;
}
}
}
$dbFarm = self::create($name, $user, $envId);
$dbFarm->createdByUserId = $user->id;
$dbFarm->createdByUserEmail = $user->getEmail();
$dbFarm->RolesLaunchOrder = $definition->rolesLaunchOrder;
$dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $definition->settings[DBFarm::SETTING_TIMEZONE]);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $definition->settings[DBFarm::SETTING_EC2_VPC_ID]);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $definition->settings[DBFarm::SETTING_EC2_VPC_REGION]);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY, $definition->settings[DBFarm::SETTING_SZR_UPD_REPOSITORY]);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE, $definition->settings[DBFarm::SETTING_SZR_UPD_SCHEDULE]);
$dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, $definition->settings[DBFarm::SETTING_LEASE_STATUS]);
$variables = new Scalr_Scripting_GlobalVariables($dbFarm->ClientID, $envId, Scalr_Scripting_GlobalVariables::SCOPE_FARM);
$variables->setValues($definition->globalVariables, 0, $dbFarm->ID, 0);
foreach ($definition->roles as $index => $role) {
$dbFarmRole = $dbFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $index + 1, $role->alias);
$oldRoleSettings = $dbFarmRole->GetAllSettings();
$dbFarmRole->applyDefinition($role, true);
$newSettings = $dbFarmRole->GetAllSettings();
Scalr_Helpers_Dns::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
// Platfrom specified updates
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
\Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
$dbFarmRolesList[] = $dbFarmRole;
$usedPlatforms[$role->platform] = 1;
}
if ($usedPlatforms[SERVER_PLATFORMS::EC2]) {
\Scalr\Modules\Platforms\Ec2\Helpers\Ec2Helper::farmSave($dbFarm, $dbFarmRolesList);
}
if ($usedPlatforms[SERVER_PLATFORMS::EUCALYPTUS]) {
\Scalr\Modules\Platforms\Eucalyptus\Helpers\EucalyptusHelper::farmSave($dbFarm, $dbFarmRolesList);
}
if ($usedPlatforms[SERVER_PLATFORMS::CLOUDSTACK]) {
CloudstackHelper::farmSave($dbFarm, $dbFarmRolesList);
}
$dbFarm->save();
return $dbFarm;
}
/**
* Clones FarmRoles with settings from this Farm to a new Farm
*
* @param DBFarm $newFarm A Farm into which Roles be cloned
*
* @throws Exception
*/
public function cloneFarmRoles(DBFarm $newFarm)
{
foreach ($this->getDefinition()->roles as $idx => $role) {
$dbFarmRole = $newFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $role->launchIndex, $role->alias);
$oldRoleSettings = $dbFarmRole->GetAllSettings();
$dbFarmRole->applyDefinition($role, true);
$newSettings = $dbFarmRole->GetAllSettings();
// Platform specified updates
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
\Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
}
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
// Collect information from database
$servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/** @var DBServer $DBServer */
try {
if ($DBServer->cloudLocation) {
try {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})"));
$p = PlatformFactory::NewPlatform($DBServer->platform);
$list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED"));
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
if (!$serverInfo) {
if ($p->debugLog) {
Logger::getLogger('Openstack')->fatal($p->debugLog);
}
if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
if (PlatformFactory::isOpenstack($DBServer->platform)) {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform)));
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId));
}
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0");
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
/*
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped')
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
} catch (Exception $e) {}
*/
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform)) {
if ($DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
// if server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} else {
// If server was suspended during initialization - we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
//TODO: Depends on resume strategy need to set server status
// For Openstack we need to re-accociate IPs
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(10);
print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
/**
* Allocate and Assign Elastic IP to instance if role use it.
*
* @param HostUpEvent $event
*/
public function OnHostUp(\HostUpEvent $event)
{
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($event->DBServer);
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm
return;
}
$this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID);
$config = \Scalr::getContainer()->config;
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/* @var $DBServer \DBServer */
try {
if ($DBServer->cloudLocation) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId);
$p = PlatformFactory::NewPlatform($DBServer->platform);
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId));
continue;
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag') {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
//More debug
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache)));
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}"));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
/* @var $env \Scalr_Environment */
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
//Saving the reason why this environment is disabled
$env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]);
return;
} elseif (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
$this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
if ($openstackErrorState) {
try {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId));
} catch (Exception $e) {
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer);
$DBServer->Save();
}
}
if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) {
$DBServer->status = \SERVER_STATUS::PENDING;
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1);
$DBServer->dateAdded = date("Y-m-d H:i:s");
$DBServer->Save();
} else {
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
\Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
}
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) {
$DBServer->remoteIp = $ipaddresses['remoteIp'];
$DBServer->localIp = $ipaddresses['localIp'];
$DBServer->Save();
}
//Add tags
Ec2Helper::createServerTags($DBServer);
}
if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scripting_event, $DBServer->farmRoleId]);
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
} elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) {
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//The list of the suspension information about cloud platforms
$this->aSuspensionInfo = [];
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform);
$jobStartTime = microtime(true);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers\n WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm and platform
return;
}
$this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform);
$config = \Scalr::getContainer()->config;
/*
if ($request->platform) {
$p = PlatformFactory::NewPlatform($request->platform);
$p->ClearCache();
}
*/
$p = PlatformFactory::NewPlatform($request->platform);
foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) {
/* @var $DBServer \DBServer */
//Get platform suspension info
$suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId);
//If the cloud platform is suspended we should not process it
if ($suspensionInfo->isSuspended()) {
continue;
}
try {
//1. We need to check that server is exists in cloud and not missed.
// (On Openstack server can be missed and should not be terminated)
$cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation);
if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
//For Azure we need to pass resource group instead of cloudLocation
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP));
} else {
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
}
//We successfully polled cloud so can resume suspension status for the cloud platform
if ($suspensionInfo->isPendingSuspend()) {
$suspensionInfo->resume();
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
continue;
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId));
continue;
}
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]);
} else {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
// If openstack server is in ERROR state we need more details
if ($openstackErrorState) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$status = empty($info['Status']) ? false : $info['Status'];
} catch (Exception $e) {
}
}
if (empty($status)) {
$status = $DBServer->GetRealStatus()->getName();
}
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
if ($DBServer->status != \SERVER_STATUS::RESUMING) {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} else {
// Need to clear cache, because this situation happens only when cache is stale.
$p->ClearCache();
}
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
$update = [];
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$update['status'] = \SERVER_STATUS::RESUMING;
$update['dateAdded'] = date("Y-m-d H:i:s");
$DBServer->update($update);
unset($update);
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) {
$elasticIpAssigned = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
$elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($elasticIpAddress) {
$ipaddresses['remoteIp'] = $elasticIpAddress;
$DBServer->remoteIp = $elasticIpAddress;
$elasticIpAssigned = true;
}
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) {
$DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]);
}
//Add tags
Ec2Helper::createObjectTags($DBServer);
}
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
// Check scalarizr deployment status
$env = $DBServer->GetEnvironmentObject();
$azure = $env->azure();
$info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME));
$extensions = !empty($info->extensions) ? $info->extensions : [];
foreach ($extensions as $extension) {
/* @var $extension ExtensionData */
if ($extension->name == 'scalarizr') {
$extStatus = $extension->statuses[0];
/* @var $extStatus StatusData */
if ($extStatus->level == 'Error') {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]);
}
}
}
}
} else {
AzureHelper::setupScalrAgent($DBServer);
}
}
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
\Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scriptingEvent = false;
$eventName = null;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$eventName = "hostInit";
$scriptingEvent = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$eventName = "hostUp";
$scriptingEvent = EVENT_TYPE::HOST_UP;
}
if ($scriptingEvent && $dtadded) {
$hasPendingMessages = !!$db->GetOne("\n SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n ", [$DBServer->serverId]);
$scriptingTimeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scriptingEvent, $DBServer->farmRoleId]);
if ($scriptingTimeout) {
$launchTimeout = $launchTimeout + $scriptingTimeout;
}
if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting() && !$elasticIpAssigned) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
//TODO: Terminated outside scalr while in SUSPENDED state
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
// Private IP cannot be removed (only changed).
if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null;
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() > $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
$this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime);
return $request;
}
public function xBuildAction()
{
$this->request->defineParams(array('farmId' => array('type' => 'int'), 'roles' => array('type' => 'json'), 'farm' => array('type' => 'json'), 'roleUpdate' => array('type' => 'int'), 'launch' => array('type' => 'bool')));
if (!$this->isFarmConfigurationValid($this->getParam('farmId'), $this->getParam('farm'), (array) $this->getParam('roles'))) {
if ($this->errors['error_count'] != 0) {
$this->response->failure();
$this->response->data(array('errors' => $this->errors));
return;
}
}
$farm = $this->getParam('farm');
$client = Client::Load($this->user->getAccountId());
if ($this->getParam('farmId')) {
$dbFarm = DBFarm::LoadByID($this->getParam('farmId'));
$this->user->getPermissions()->validate($dbFarm);
$this->request->restrictFarmAccess($dbFarm, Acl::PERM_FARMS_MANAGE);
$dbFarm->isLocked();
if ($this->getParam('changed') && $dbFarm->changedTime && $this->getParam('changed') != $dbFarm->changedTime) {
$userName = '******';
$changed = explode(' ', $this->getParam('changed'));
$changedTime = intval($changed[1]);
try {
$user = new Scalr_Account_User();
$user->loadById($dbFarm->changedByUserId);
$userName = $user->getEmail();
} catch (Exception $e) {
}
$this->response->failure();
$this->response->data(array('changedFailure' => sprintf('%s changed this farm at %s', $userName, Scalr_Util_DateTime::convertTz($changedTime))));
return;
}
$dbFarm->changedByUserId = $this->user->getId();
$dbFarm->changedTime = microtime();
$bNew = false;
} else {
$this->request->restrictFarmAccess(null, Acl::PERM_FARMS_MANAGE);
$this->user->getAccount()->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
$dbFarm = new DBFarm();
$dbFarm->ClientID = $this->user->getAccountId();
$dbFarm->EnvID = $this->getEnvironmentId();
$dbFarm->Status = FARM_STATUS::TERMINATED;
$dbFarm->createdByUserId = $this->user->getId();
$dbFarm->createdByUserEmail = $this->user->getEmail();
$dbFarm->changedByUserId = $this->user->getId();
$dbFarm->changedTime = microtime();
$bNew = true;
}
if ($this->getParam('farm')) {
$dbFarm->Name = $this->request->stripValue($farm['name']);
$dbFarm->RolesLaunchOrder = $farm['rolesLaunchOrder'];
$dbFarm->Comments = $this->request->stripValue($farm['description']);
}
if (empty($dbFarm->Name)) {
throw new Exception(_("Farm name required"));
}
if ($bNew) {
$dbFarm->teamId = is_numeric($farm['teamOwner']) && $farm['teamOwner'] > 0 ? $farm['teamOwner'] : NULL;
} else {
if ($dbFarm->createdByUserId == $this->user->getId() || $this->user->isAccountOwner() || $this->request->isFarmAllowed($dbFarm, Acl::PERM_FARMS_CHANGE_OWNERSHIP)) {
if (is_numeric($farm['owner']) && $farm['owner'] != $dbFarm->createdByUserId) {
$user = (new Scalr_Account_User())->loadById($farm['owner']);
$dbFarm->createdByUserId = $user->getId();
$dbFarm->createdByUserEmail = $user->getEmail();
// TODO: move to subclass \Farm\Setting\OwnerHistory
$history = unserialize($dbFarm->GetSetting(DBFarm::SETTING_OWNER_HISTORY));
if (!is_array($history)) {
$history = [];
}
$history[] = ['newId' => $user->getId(), 'newEmail' => $user->getEmail(), 'changedById' => $this->user->getId(), 'changedByEmail' => $this->user->getEmail(), 'dt' => date('Y-m-d H:i:s')];
$dbFarm->SetSetting(DBFarm::SETTING_OWNER_HISTORY, serialize($history));
}
$dbFarm->teamId = is_numeric($farm['teamOwner']) && $farm['teamOwner'] > 0 ? $farm['teamOwner'] : NULL;
}
}
$dbFarm->save();
$governance = new Scalr_Governance($this->getEnvironmentId());
if (!$this->getParam('farmId') && $governance->isEnabled(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE)) {
$dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, 'Active');
// for created farm
}
if (isset($farm['variables'])) {
$variables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), Scalr_Scripting_GlobalVariables::SCOPE_FARM);
$variables->setValues(is_array($farm['variables']) ? $farm['variables'] : [], 0, $dbFarm->ID, 0, '', false, true);
}
if (!$farm['timezone']) {
$farm['timezone'] = date_default_timezone_get();
}
$dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $farm['timezone']);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $farm['vpc_id']);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $farm['vpc_region']);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY, $farm[DBFarm::SETTING_SZR_UPD_REPOSITORY]);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE, $farm[DBFarm::SETTING_SZR_UPD_SCHEDULE]);
if (!$dbFarm->GetSetting(DBFarm::SETTING_CRYPTO_KEY)) {
$dbFarm->SetSetting(DBFarm::SETTING_CRYPTO_KEY, Scalr::GenerateRandomKey(40));
}
if ($this->getContainer()->analytics->enabled) {
//Cost analytics project must be set for the Farm object
$dbFarm->setProject(!empty($farm['projectId']) ? $farm['projectId'] : null);
}
$virtualFarmRoles = array();
$roles = $this->getParam('roles');
if (!empty($roles)) {
foreach ($roles as $role) {
if (strpos($role['farm_role_id'], "virtual_") !== false) {
$dbRole = DBRole::loadById($role['role_id']);
$dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index'], $role['alias']);
$virtualFarmRoles[$role['farm_role_id']] = $dbFarmRole->ID;
}
}
}
$usedPlatforms = array();
$dbFarmRolesList = array();
$newFarmRolesList = array();
$farmRoleVariables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), Scalr_Scripting_GlobalVariables::SCOPE_FARMROLE);
if (!empty($roles)) {
foreach ($roles as $role) {
if ($role['farm_role_id']) {
if ($virtualFarmRoles[$role['farm_role_id']]) {
$role['farm_role_id'] = $virtualFarmRoles[$role['farm_role_id']];
}
$update = true;
$dbFarmRole = DBFarmRole::LoadByID($role['farm_role_id']);
$dbRole = DBRole::loadById($dbFarmRole->RoleID);
$role['role_id'] = $dbFarmRole->RoleID;
if ($dbFarmRole->Platform == SERVER_PLATFORMS::GCE) {
$dbFarmRole->CloudLocation = $role['cloud_location'];
}
} else {
$update = false;
$dbRole = DBRole::loadById($role['role_id']);
$dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index']);
}
if ($dbRole->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$role['settings'][DBFarmRole::SETTING_SCALING_MAX_INSTANCES] = $role['settings'][DBFarmRole::SETTING_SCALING_MIN_INSTANCES];
}
if ($dbFarmRole->NewRoleID) {
continue;
}
if ($update) {
$dbFarmRole->LaunchIndex = (int) $role['launch_index'];
$dbFarmRole->Alias = $role['alias'];
$dbFarmRole->Save();
}
$usedPlatforms[$role['platform']] = 1;
$oldRoleSettings = $dbFarmRole->GetAllSettings();
// Update virtual farm_role_id with actual value
$scripts = (array) $role['scripting'];
if (count($virtualFarmRoles) > 0) {
array_walk_recursive($scripts, function (&$v, $k) use($virtualFarmRoles) {
if (is_string($v)) {
$v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
}
});
array_walk_recursive($role['settings'], function (&$v, $k) use($virtualFarmRoles) {
if (is_string($v)) {
$v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
}
});
}
$dbFarmRole->ClearSettings("chef.");
if (!empty($role['scaling_settings']) && is_array($role['scaling_settings'])) {
foreach ($role['scaling_settings'] as $k => $v) {
$dbFarmRole->SetSetting($k, $v, DBFarmRole::TYPE_CFG);
}
}
foreach ($role['settings'] as $k => $v) {
$dbFarmRole->SetSetting($k, $v, DBFarmRole::TYPE_CFG);
}
/****** Scaling settings ******/
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingManager->setFarmRoleMetrics(is_array($role['scaling']) ? $role['scaling'] : array());
//TODO: optimize this code...
$this->db->Execute("DELETE FROM farm_role_scaling_times WHERE farm_roleid=?", array($dbFarmRole->ID));
// 5 = Time based scaling -> move to constants
if ($role['scaling'][5]) {
foreach ($role['scaling'][5] as $scal_period) {
$chunks = explode(":", $scal_period['id']);
$this->db->Execute("INSERT INTO farm_role_scaling_times SET\n farm_roleid\t\t= ?,\n start_time\t\t= ?,\n end_time\t\t= ?,\n days_of_week\t= ?,\n instances_count\t= ?\n ", array($dbFarmRole->ID, $chunks[0], $chunks[1], $chunks[2], $chunks[3]));
}
}
/*****************/
/* Update role params */
$dbFarmRole->SetParameters((array) $role['params']);
/* End of role params management */
/* Add script options to databse */
$dbFarmRole->SetScripts($scripts, (array) $role['scripting_params']);
/* End of scripting section */
/* Add services configuration */
$dbFarmRole->SetServiceConfigPresets((array) $role['config_presets']);
/* End of scripting section */
/* Add storage configuration */
if (isset($role['storages'])) {
if (isset($role['storages']['configs'])) {
$dbFarmRole->getStorage()->setConfigs($role['storages']['configs']);
}
}
$farmRoleVariables->setValues(is_array($role['variables']) ? $role['variables'] : [], $dbFarmRole->GetRoleID(), $dbFarm->ID, $dbFarmRole->ID, '', false, true);
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->onFarmSave($dbFarm, $dbFarmRole);
}
/**
* Platform specified updates
*/
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
\Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
\Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
}
if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
Scalr\Modules\Platforms\Cloudstack\Helpers\CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
}
$dbFarmRolesList[] = $dbFarmRole;
$newFarmRolesList[] = $dbFarmRole->ID;
}
}
if (!$this->getParam('roleUpdate')) {
foreach ($dbFarm->GetFarmRoles() as $dbFarmRole) {
if (!$dbFarmRole->NewRoleID && !in_array($dbFarmRole->ID, $newFarmRolesList)) {
$dbFarmRole->Delete();
}
}
}
$dbFarm->save();
if (!$client->GetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED)) {
$client->SetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED, time());
}
if ($this->request->isFarmAllowed($dbFarm, Acl::PERM_FARMS_LAUNCH_TERMINATE) && $this->getParam('launch')) {
$this->user->getPermissions()->validate($dbFarm);
$dbFarm->isLocked();
Scalr::FireEvent($dbFarm->ID, new FarmLaunchedEvent(true, $this->user->id));
$this->response->success('Farm successfully saved and launched');
} else {
$this->response->success('Farm successfully saved');
}
$this->response->data(array('farmId' => $dbFarm->ID, 'isNewFarm' => $bNew));
}
/**
* Creates clone for the farm
*
* @param string $name The name of the farm
* @param Scalr_Account_User $user The user object
* @param int $envId The identifier of the environment
* @return DBFarm Returns clone
*/
public function cloneFarm($name = false, Scalr_Account_User $user, $envId)
{
$account = $user->getAccount();
$account->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
$definition = $this->getDefinition();
if (!$name) {
$template = "";
if (!preg_match('/^(.*?)\\(clone \\#([0-9]*)\\)$/si', $definition->name)) {
$name = $definition->name;
} else {
preg_match('/^(.*?)\\(clone \\#([0-9]*)\\)$/si', $definition->name, $matches);
$name = trim($matches[1]);
}
$name = preg_replace('/[^A-Za-z0-9_\\. -]+/', '', $name);
$lastUsedIndex = $this->DB->GetOne('SELECT MAX(CAST((SUBSTR(@p:=SUBSTRING_INDEX(name, \'#\', -1), 1, LENGTH(@p) - 1)) AS UNSIGNED)) as lastUsedCloneNumber FROM farms WHERE name REGEXP \'' . str_replace('.', '\\.', $name) . ' \\\\(clone #[0-9]+\\\\)\' AND env_id = ?', array($envId));
$name = $name . ' (clone #' . ($lastUsedIndex + 1) . ')';
}
$dbFarm = self::create($name, $user, $envId);
$dbFarm->createdByUserId = $user->id;
$dbFarm->createdByUserEmail = $user->getEmail();
$dbFarm->RolesLaunchOrder = $definition->rolesLaunchOrder;
$dbFarm->teamId = $definition->teamId;
$dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $definition->settings[DBFarm::SETTING_TIMEZONE]);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $definition->settings[DBFarm::SETTING_EC2_VPC_ID]);
$dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $definition->settings[DBFarm::SETTING_EC2_VPC_REGION]);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY, $definition->settings[DBFarm::SETTING_SZR_UPD_REPOSITORY]);
$dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE, $definition->settings[DBFarm::SETTING_SZR_UPD_SCHEDULE]);
$dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, $definition->settings[DBFarm::SETTING_LEASE_STATUS]);
$variables = new Scalr_Scripting_GlobalVariables($dbFarm->ClientID, $envId, Scalr_Scripting_GlobalVariables::SCOPE_FARM);
$variables->setValues($definition->globalVariables, 0, $dbFarm->ID, 0);
foreach ($definition->roles as $index => $role) {
$dbFarmRole = $dbFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $index + 1, $role->alias);
$oldRoleSettings = $dbFarmRole->GetAllSettings();
$dbFarmRole->applyDefinition($role, true);
$newSettings = $dbFarmRole->GetAllSettings();
// Platform specified updates
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
\Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
\Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
}
$dbFarmRolesList[] = $dbFarmRole;
$usedPlatforms[$role->platform] = 1;
}
$dbFarm->save();
return $dbFarm;
}
public function xBuildAction()
{
$this->request->defineParams(array('farmId' => array('type' => 'int'), 'roles' => array('type' => 'json'), 'rolesToRemove' => array('type' => 'json'), 'farm' => array('type' => 'json'), 'launch' => array('type' => 'bool')));
if (!$this->isFarmConfigurationValid($this->getParam('farmId'), $this->getParam('farm'), (array) $this->getParam('roles'))) {
if ($this->errors['error_count'] != 0) {
$this->response->failure();
$this->response->data(array('errors' => $this->errors));
return;
}
}
$farm = $this->getParam('farm');
$client = Client::Load($this->user->getAccountId());
if ($this->getParam('farmId')) {
$dbFarm = DBFarm::LoadByID($this->getParam('farmId'));
$this->user->getPermissions()->validate($dbFarm);
$this->request->checkPermissions($dbFarm->__getNewFarmObject(), Acl::PERM_FARMS_UPDATE);
$dbFarm->isLocked();
if ($this->getParam('changed') && $dbFarm->changedTime && $this->getParam('changed') != $dbFarm->changedTime) {
$userName = '******';
$changed = explode(' ', $this->getParam('changed'));
$changedTime = intval($changed[1]);
try {
$user = new Scalr_Account_User();
$user->loadById($dbFarm->changedByUserId);
$userName = $user->getEmail();
} catch (Exception $e) {
}
$this->response->failure();
$this->response->data(array('changedFailure' => sprintf('%s changed this farm at %s', $userName, Scalr_Util_DateTime::convertTz($changedTime))));
return;
} else {
if ($this->getParam('changed')) {
$this->checkFarmConfigurationIntegrity($this->getParam('farmId'), $this->getParam('farm'), (array) $this->getParam('roles'), (array) $this->getParam('rolesToRemove'));
}
}
$dbFarm->changedByUserId = $this->user->getId();
$dbFarm->changedTime = microtime();
if ($this->getContainer()->analytics->enabled) {
$projectId = $farm['projectId'];
if (empty($projectId)) {
$ccId = $dbFarm->GetEnvironmentObject()->getPlatformConfigValue(Scalr_Environment::SETTING_CC_ID);
if (!empty($ccId)) {
//Assigns Project automatically only if it is the one withing the Cost Center
$projects = ProjectEntity::findByCcId($ccId);
if (count($projects) == 1) {
$projectId = $projects->getArrayCopy()[0]->projectId;
}
}
}
if (!empty($projectId) && $dbFarm->GetSetting(Entity\FarmSetting::PROJECT_ID) != $projectId) {
$this->request->checkPermissions($dbFarm->__getNewFarmObject(), Acl::PERM_FARMS_PROJECTS);
}
}
$bNew = false;
} else {
$this->request->restrictAccess(Acl::RESOURCE_OWN_FARMS, Acl::PERM_FARMS_CREATE);
$this->user->getAccount()->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
$dbFarm = new DBFarm();
$dbFarm->ClientID = $this->user->getAccountId();
$dbFarm->EnvID = $this->getEnvironmentId();
$dbFarm->Status = FARM_STATUS::TERMINATED;
$dbFarm->ownerId = $this->user->getId();
$dbFarm->changedByUserId = $this->user->getId();
$dbFarm->changedTime = microtime();
$bNew = true;
}
if ($this->getParam('farm')) {
$dbFarm->Name = $this->request->stripValue($farm['name']);
$dbFarm->RolesLaunchOrder = $farm['rolesLaunchOrder'];
$dbFarm->Comments = $this->request->stripValue($farm['description']);
}
if (empty($dbFarm->Name)) {
throw new Exception(_("Farm name required"));
}
$setFarmTeams = false;
if ($bNew) {
$setFarmTeams = true;
} else {
if ($dbFarm->ownerId == $this->user->getId() || $this->request->hasPermissions($dbFarm->__getNewFarmObject(), Acl::PERM_FARMS_CHANGE_OWNERSHIP)) {
if (is_numeric($farm['owner']) && $farm['owner'] != $dbFarm->ownerId) {
$dbFarm->ownerId = $farm['owner'];
$f = Entity\Farm::findPk($dbFarm->ID);
Entity\FarmSetting::addOwnerHistory($f, User::findPk($farm['owner']), User::findPk($this->user->getId()));
$f->save();
}
$setFarmTeams = true;
}
}
$dbFarm->save();
if ($setFarmTeams && is_array($farm['teamOwner'])) {
/* @var $f Entity\Farm */
$f = Entity\Farm::findPk($dbFarm->ID);
$f->setTeams(empty($farm['teamOwner']) ? [] : Entity\Account\Team::find([['name' => ['$in' => $farm['teamOwner']]], ['accountId' => $this->getUser()->accountId]]));
$f->save();
}
if ($bNew) {
$dbFarm->SetSetting(Entity\FarmSetting::CREATED_BY_ID, $this->user->getId());
$dbFarm->SetSetting(Entity\FarmSetting::CREATED_BY_EMAIL, $this->user->getEmail());
}
$governance = new Scalr_Governance($this->getEnvironmentId());
if (!$this->getParam('farmId') && $governance->isEnabled(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE)) {
$dbFarm->SetSetting(Entity\FarmSetting::LEASE_STATUS, 'Active');
// for created farm
}
if (isset($farm['variables'])) {
$variables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), ScopeInterface::SCOPE_FARM);
$variables->setValues(is_array($farm['variables']) ? $farm['variables'] : [], 0, $dbFarm->ID, 0, '', false, true);
}
if (!$farm['timezone']) {
$farm['timezone'] = date_default_timezone_get();
}
$dbFarm->SetSetting(Entity\FarmSetting::TIMEZONE, $farm['timezone']);
$dbFarm->SetSetting(Entity\FarmSetting::EC2_VPC_ID, isset($farm["vpc_id"]) ? $farm['vpc_id'] : null);
$dbFarm->SetSetting(Entity\FarmSetting::EC2_VPC_REGION, isset($farm["vpc_id"]) ? $farm['vpc_region'] : null);
$dbFarm->SetSetting(Entity\FarmSetting::SZR_UPD_REPOSITORY, $farm[Entity\FarmSetting::SZR_UPD_REPOSITORY]);
$dbFarm->SetSetting(Entity\FarmSetting::SZR_UPD_SCHEDULE, $farm[Entity\FarmSetting::SZR_UPD_SCHEDULE]);
if (!$dbFarm->GetSetting(Entity\FarmSetting::CRYPTO_KEY)) {
$dbFarm->SetSetting(Entity\FarmSetting::CRYPTO_KEY, Scalr::GenerateRandomKey(40));
}
if ($this->getContainer()->analytics->enabled) {
//Cost analytics project must be set for the Farm object
$dbFarm->setProject(!empty($farm['projectId']) ? $farm['projectId'] : null);
}
$virtualFarmRoles = array();
$roles = $this->getParam('roles');
if (!empty($roles)) {
foreach ($roles as $role) {
if (strpos($role['farm_role_id'], "virtual_") !== false) {
$dbRole = DBRole::loadById($role['role_id']);
$dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index'], $role['alias']);
$virtualFarmRoles[$role['farm_role_id']] = $dbFarmRole->ID;
}
}
}
$usedPlatforms = array();
$farmRoleVariables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), ScopeInterface::SCOPE_FARMROLE);
if (!empty($roles)) {
foreach ($roles as $role) {
if ($role['farm_role_id']) {
if (isset($virtualFarmRoles[$role['farm_role_id']])) {
$role['farm_role_id'] = $virtualFarmRoles[$role['farm_role_id']];
}
$update = true;
$dbFarmRole = DBFarmRole::LoadByID($role['farm_role_id']);
$dbRole = DBRole::loadById($dbFarmRole->RoleID);
$role['role_id'] = $dbFarmRole->RoleID;
if ($dbFarmRole->Platform == SERVER_PLATFORMS::GCE) {
$dbFarmRole->CloudLocation = $role['cloud_location'];
}
} else {
/** TODO: Remove because will be handled with virtual_ **/
$update = false;
$dbRole = DBRole::loadById($role['role_id']);
$dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index']);
}
if ($dbRole->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$role['settings'][Entity\FarmRoleSetting::SCALING_MAX_INSTANCES] = $role['settings'][Entity\FarmRoleSetting::SCALING_MIN_INSTANCES];
}
if ($update) {
$dbFarmRole->LaunchIndex = (int) $role['launch_index'];
$dbFarmRole->Alias = $role['alias'];
$dbFarmRole->Save();
}
$usedPlatforms[$role['platform']] = 1;
$oldRoleSettings = $dbFarmRole->GetAllSettings();
// Update virtual farm_role_id with actual value
$scripts = (array) $role['scripting'];
if (!empty($virtualFarmRoles)) {
array_walk_recursive($scripts, function (&$v, $k) use($virtualFarmRoles) {
if (is_string($v)) {
$v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
}
});
array_walk_recursive($role['settings'], function (&$v, $k) use($virtualFarmRoles) {
if (is_string($v)) {
$v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
}
});
}
$dbFarmRole->ClearSettings("chef.");
if (!empty($role['scaling_settings']) && is_array($role['scaling_settings'])) {
foreach ($role['scaling_settings'] as $k => $v) {
$dbFarmRole->SetSetting($k, $v, Entity\FarmRoleSetting::TYPE_CFG);
}
}
foreach ($role['settings'] as $k => $v) {
$dbFarmRole->SetSetting($k, $v, Entity\FarmRoleSetting::TYPE_CFG);
}
/****** Scaling settings ******/
$scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
$scalingManager->setFarmRoleMetrics(is_array($role['scaling']) ? $role['scaling'] : array());
//TODO: optimize this code...
$this->db->Execute("DELETE FROM farm_role_scaling_times WHERE farm_roleid=?", array($dbFarmRole->ID));
// 5 = Time based scaling -> move to constants
if (!empty($role['scaling'][Entity\ScalingMetric::METRIC_DATE_AND_TIME_ID])) {
foreach ($role['scaling'][Entity\ScalingMetric::METRIC_DATE_AND_TIME_ID] as $scal_period) {
$chunks = explode(":", $scal_period['id']);
$this->db->Execute("INSERT INTO farm_role_scaling_times SET\n farm_roleid\t\t= ?,\n start_time\t\t= ?,\n end_time\t\t= ?,\n days_of_week\t= ?,\n instances_count\t= ?\n ", array($dbFarmRole->ID, $chunks[0], $chunks[1], $chunks[2], $chunks[3]));
}
}
/*****************/
/* Add script options to databse */
$dbFarmRole->SetScripts($scripts, (array) $role['scripting_params']);
/* End of scripting section */
/* Add storage configuration */
if (isset($role['storages']['configs'])) {
$dbFarmRole->getStorage()->setConfigs($role['storages']['configs'], false);
}
$farmRoleVariables->setValues(is_array($role['variables']) ? $role['variables'] : [], $dbFarmRole->GetRoleID(), $dbFarm->ID, $dbFarmRole->ID, '', false, true);
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->onFarmSave($dbFarm, $dbFarmRole);
}
/**
* Platform specified updates
*/
if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
\Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
\Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
if ($role['settings']['aws.elb.remove']) {
$this->request->restrictAccess(Acl::RESOURCE_AWS_ELB, Acl::PERM_AWS_ELB_MANAGE);
}
\Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
}
if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
Scalr\Modules\Platforms\Cloudstack\Helpers\CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
}
}
}
$rolesToRemove = $this->getParam('rolesToRemove');
if (!empty($rolesToRemove)) {
$currentFarmRoles = Entity\FarmRole::find([['farmId' => $dbFarm->ID], ['id' => ['$in' => $rolesToRemove]]]);
/* @var $farmRole Entity\FarmRole */
foreach ($currentFarmRoles as $farmRole) {
$farmRole->delete();
}
}
$dbFarm->save();
if (!$client->GetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED)) {
$client->SetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED, time());
}
if ($this->request->hasPermissions($dbFarm->__getNewFarmObject(), Acl::PERM_FARMS_LAUNCH_TERMINATE) && $this->getParam('launch')) {
$this->user->getPermissions()->validate($dbFarm);
$dbFarm->isLocked();
Scalr::FireEvent($dbFarm->ID, new FarmLaunchedEvent(true, $this->user->id));
$this->response->success('Farm successfully saved and launched');
} else {
$this->response->success('Farm successfully saved');
}
$this->response->data(array('farmId' => $dbFarm->ID, 'isNewFarm' => $bNew));
}
