Example #1
0
 /**
  * Creates clone for the farm
  *
  * @param   string             $name   The name of the farm
  * @param   Scalr_Account_User $user   The user object
  * @param   int                $envId  The identifier of the environment
  * @return  DBFarm             Returns clone
  */
 public function cloneFarm($name = false, Scalr_Account_User $user, $envId)
 {
     $account = $user->getAccount();
     $account->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
     $definition = $this->getDefinition();
     if (!$name) {
         $template = "";
         if (!stristr($definition->name, "clone")) {
             $template = $definition->name . ' (clone #%s)';
             $i = 1;
         } else {
             preg_match("/^(.*?)\\(clone \\#([0-9]*)\\)\$/si", $definition->name, $matches);
             $template = trim($matches[1]) . " (clone #%s)";
             $i = $matches[2] + 1;
         }
         while (true) {
             $name = sprintf($template, $i);
             if (!$this->DB->GetOne("SELECT id FROM farms WHERE name = ? AND env_id = ? LIMIT 1", array($name, $this->EnvID))) {
                 break;
             } else {
                 $i++;
             }
         }
     }
     $dbFarm = self::create($name, $user, $envId);
     $dbFarm->createdByUserId = $user->id;
     $dbFarm->createdByUserEmail = $user->getEmail();
     $dbFarm->RolesLaunchOrder = $definition->rolesLaunchOrder;
     $dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $definition->settings[DBFarm::SETTING_TIMEZONE]);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $definition->settings[DBFarm::SETTING_EC2_VPC_ID]);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $definition->settings[DBFarm::SETTING_EC2_VPC_REGION]);
     $dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY, $definition->settings[DBFarm::SETTING_SZR_UPD_REPOSITORY]);
     $dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE, $definition->settings[DBFarm::SETTING_SZR_UPD_SCHEDULE]);
     $dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, $definition->settings[DBFarm::SETTING_LEASE_STATUS]);
     $variables = new Scalr_Scripting_GlobalVariables($dbFarm->ClientID, $envId, Scalr_Scripting_GlobalVariables::SCOPE_FARM);
     $variables->setValues($definition->globalVariables, 0, $dbFarm->ID, 0);
     foreach ($definition->roles as $index => $role) {
         $dbFarmRole = $dbFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $index + 1, $role->alias);
         $oldRoleSettings = $dbFarmRole->GetAllSettings();
         $dbFarmRole->applyDefinition($role, true);
         $newSettings = $dbFarmRole->GetAllSettings();
         Scalr_Helpers_Dns::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         // Platfrom specified updates
         if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
             \Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
         if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
             CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
         $dbFarmRolesList[] = $dbFarmRole;
         $usedPlatforms[$role->platform] = 1;
     }
     if ($usedPlatforms[SERVER_PLATFORMS::EC2]) {
         \Scalr\Modules\Platforms\Ec2\Helpers\Ec2Helper::farmSave($dbFarm, $dbFarmRolesList);
     }
     if ($usedPlatforms[SERVER_PLATFORMS::EUCALYPTUS]) {
         \Scalr\Modules\Platforms\Eucalyptus\Helpers\EucalyptusHelper::farmSave($dbFarm, $dbFarmRolesList);
     }
     if ($usedPlatforms[SERVER_PLATFORMS::CLOUDSTACK]) {
         CloudstackHelper::farmSave($dbFarm, $dbFarmRolesList);
     }
     $dbFarm->save();
     return $dbFarm;
 }
Example #2
0
 /**
  * Clones FarmRoles with settings from this Farm to a new Farm
  *
  * @param   DBFarm  $newFarm    A Farm into which Roles be cloned
  *
  * @throws  Exception
  */
 public function cloneFarmRoles(DBFarm $newFarm)
 {
     foreach ($this->getDefinition()->roles as $idx => $role) {
         $dbFarmRole = $newFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $role->launchIndex, $role->alias);
         $oldRoleSettings = $dbFarmRole->GetAllSettings();
         $dbFarmRole->applyDefinition($role, true);
         $newSettings = $dbFarmRole->GetAllSettings();
         // Platform specified updates
         if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
             \Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
         if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
             CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
     }
 }
 /**
  * {@inheritdoc}
  * @see \Scalr\Modules\PlatformModuleInterface::ResumeServer()
  */
 public function ResumeServer(DBServer $DBServer)
 {
     $cs = $DBServer->GetEnvironmentObject()->cloudstack($this->platform);
     try {
         $cs->instance->start($DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
     } catch (NotFoundException $e) {
         throw new InstanceNotFoundException($e->getMessage(), $e->getCode(), $e);
     }
     $remoteIp = CloudstackHelper::getSharedIP($DBServer);
     if ($remoteIp) {
         $DBServer->remoteIp = $remoteIp;
     }
     parent::ResumeServer($DBServer);
     return true;
 }
 public function OnHostInit(\HostInitEvent $event)
 {
     if (!in_array($event->DBServer->platform, array(\SERVER_PLATFORMS::CLOUDSTACK, \SERVER_PLATFORMS::IDCF))) {
         return;
     }
     if ($event->DBServer->farmRoleId) {
         $dbFarmRole = $event->DBServer->GetFarmRoleObject();
         if ($dbFarmRole->GetSetting(\DBFarmRole::SETIING_CLOUDSTACK_USE_STATIC_NAT)) {
             CloudstackHelper::setStaticNatForServer($event->DBServer);
             return true;
         }
         $networkType = $dbFarmRole->GetSetting(\DBFarmRole::SETTING_CLOUDSTACK_NETWORK_TYPE);
         $networkId = $dbFarmRole->GetSetting(\DBFarmRole::SETTING_CLOUDSTACK_NETWORK_ID);
         if ($networkType == 'Direct' || !$networkId) {
             return true;
         }
         if ($networkId == 'SCALR_MANUAL') {
             $map = $dbFarmRole->GetSetting(\DBFarmRole::SETIING_CLOUDSTACK_STATIC_NAT_PRIVATE_MAP);
             $map = explode(";", $map);
             foreach ($map as $ipMapping) {
                 $ipInfo = explode("=", $ipMapping);
                 if ($ipInfo[0] == $event->DBServer->localIp) {
                     $event->DBServer->remoteIp = $ipInfo[1];
                     $event->DBServer->Save();
                 }
             }
         }
         $sharedIpId = $dbFarmRole->GetSetting(\DBFarmRole::SETTING_CLOUDSTACK_SHARED_IP_ID);
     }
     $platform = PlatformFactory::NewPlatform($event->DBServer->platform);
     try {
         $environment = $event->DBServer->GetEnvironmentObject();
         $cloudLocation = $event->DBServer->GetCloudLocation();
         if (!$sharedIpId) {
             $sharedIpId = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP_ID . ".{$cloudLocation}", $environment, false);
         }
         if (!$sharedIpId) {
             return true;
         }
         $cs = $environment->cloudstack($event->DBServer->platform);
         // Create port forwarding rules for scalarizr
         $port = $platform->getConfigVariable(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false);
         if (!$port) {
             $port1 = 30000;
             $port2 = 30001;
             $port3 = 30002;
             $port4 = 30003;
         } else {
             $port1 = $port + 1;
             $port2 = $port1 + 1;
             $port3 = $port2 + 1;
             $port4 = $port3 + 1;
         }
         $result2 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8014, 'protocol' => "udp", 'publicport' => $port1, 'virtualmachineid' => $event->DBServer->GetProperty(\CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)));
         $result1 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8013, 'protocol' => "tcp", 'publicport' => $port1, 'virtualmachineid' => $event->DBServer->GetProperty(\CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)));
         $result3 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8010, 'protocol' => "tcp", 'publicport' => $port3, 'virtualmachineid' => $event->DBServer->GetProperty(\CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)));
         $result4 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8008, 'protocol' => "tcp", 'publicport' => $port2, 'virtualmachineid' => $event->DBServer->GetProperty(\CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)));
         $result5 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 22, 'protocol' => "tcp", 'publicport' => $port4, 'virtualmachineid' => $event->DBServer->GetProperty(\CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)));
         $event->DBServer->SetProperties(array(\SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, \SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, \SERVER_PROPERTIES::SZR_API_PORT => $port3, \SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, \SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4));
         $platform->setConfigVariable(array(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false);
     } catch (\Exception $e) {
         $this->Logger->fatal(new \FarmLogMessage($this->FarmID, sprintf(_("Cloudstack handler failed: %s."), $e->getMessage())));
     }
 }
 private function onHostInit($message, DBServer $dbserver)
 {
     $logger = Logger::getLogger(__CLASS__);
     if ($dbserver->status == SERVER_STATUS::PENDING) {
         $platform = PlatformFactory::NewPlatform($dbserver->platform);
         // Update server crypto key
         $srv_props = array();
         if ($message->cryptoKey) {
             $srv_props[SERVER_PROPERTIES::SZR_KEY] = trim($message->cryptoKey);
             $srv_props[SERVER_PROPERTIES::SZR_KEY_TYPE] = SZR_KEY_TYPE::PERMANENT;
         }
         $srv_props[SERVER_PROPERTIES::SZR_SNMP_PORT] = $message->snmpPort;
         if ($dbserver->isCloudstack()) {
             $remoteIp = CloudstackHelper::getSharedIP($dbserver);
         }
         if ($dbserver->isOpenstack()) {
             if ($dbserver->farmRoleId) {
                 $ipPool = $dbserver->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_OPENSTACK_IP_POOL);
                 if ($ipPool && empty($dbserver->remoteIp)) {
                     return false;
                 } else {
                     $remoteIp = $dbserver->remoteIp;
                 }
             }
         }
         if (!$remoteIp) {
             $ips = $platform->GetServerIPAddresses($dbserver);
             if ($ips['remoteIp']) {
                 $remoteIp = $ips['remoteIp'];
             } else {
                 $remoteIp = $message->remoteIp ? $ips['remoteIp'] : '';
             }
         }
         $dbserver->remoteIp = $remoteIp;
         $dbserver->Save();
         //Update auto-update settings
         //TODO: Check auto-update client version
         if ($dbserver->IsSupported('0.7.225') && !$dbserver->IsSupported('2.7.10')) {
             $dbserver->SetProperties($srv_props);
             try {
                 $repo = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Role_Behavior::ROLE_BASE_SZR_UPD_REPOSITORY);
                 if (!$repo) {
                     $repo = $dbserver->GetFarmObject()->GetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY);
                 }
                 $schedule = $dbserver->GetFarmObject()->GetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE);
                 if ($repo && $schedule) {
                     $updateClient = new Scalr_Net_Scalarizr_UpdateClient($dbserver);
                     $updateClient->configure($repo, $schedule);
                 }
             } catch (Exception $e) {
             }
         }
         // MySQL specific
         $dbFarmRole = $dbserver->GetFarmRoleObject();
         if ($dbFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MYSQL)) {
             $master = $dbFarmRole->GetFarmObject()->GetMySQLInstances(true);
             // If no masters in role this server becomes it
             if (!$master[0] && !(int) $dbFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_SLAVE_TO_MASTER)) {
                 $srv_props[SERVER_PROPERTIES::DB_MYSQL_MASTER] = 1;
             }
         }
         //MSR Replication Master
         //TODO: MySQL
         if ($dbFarmRole->GetRoleObject()->getDbMsrBehavior()) {
             $servers = $dbFarmRole->GetServersByFilter(array('status' => array(SERVER_STATUS::INIT, SERVER_STATUS::RUNNING)));
             if (!$dbFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
                 $masterFound = false;
                 foreach ($servers as $server) {
                     if ($server->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER)) {
                         $masterFound = true;
                     }
                 }
                 if (!$masterFound) {
                     $srv_props[Scalr_Db_Msr::REPLICATION_MASTER] = 1;
                 }
             } elseif ($dbFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER) && count($servers) == 0) {
                 $dbFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL);
                 $srv_props[Scalr_Db_Msr::REPLICATION_MASTER] = 1;
             }
         }
         $dbserver->SetProperties($srv_props);
         return new HostInitEvent($dbserver, $message->localIp, $remoteIp, $message->sshPubKey);
     } else {
         /*
            $logger->error("Strange situation. Received HostInit message"
                    . " from server '{$dbserver->serverId}' ({$message->remoteIp})"
                    . " with state {$dbserver->status}!");
         */
         //TOOD: Check if instance terminating we probably can cancel termination and continue initialization
     }
 }
Example #6
0
 /**
  * {@inheritdoc}
  * @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
  */
 public function worker($request)
 {
     $db = \Scalr::getDb();
     //Speed up poller
     if ($this->config()->daemon) {
         //Warming up static DI cache
         \Scalr::getContainer()->warmup();
     }
     // Reconfigure observers
     \Scalr::ReconfigureObservers();
     $DBFarm = DBFarm::LoadByID($request->farmId);
     $account = Scalr_Account::init()->loadById($DBFarm->ClientID);
     $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
     $transactionId = abs(crc32(posix_getpid() . $request->farmId));
     $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status);
     //Retrieves the number of either terminated or suspended servers for the farm
     $servers_count = $db->GetOne("\n            SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n        ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
     if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
         //There are no servers for this farm
         return;
     }
     $this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID);
     $config = \Scalr::getContainer()->config;
     foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
         /* @var $DBServer \DBServer */
         try {
             if ($DBServer->cloudLocation) {
                 try {
                     $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId);
                     $p = PlatformFactory::NewPlatform($DBServer->platform);
                     $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
                 } catch (Exception $e) {
                     $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
                     continue;
                 }
             }
             if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
                 if (!$p->IsServerExists($DBServer)) {
                     try {
                         $serverInfo = $p->GetServerExtendedInformation($DBServer);
                     } catch (Exception $e) {
                         $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
                     }
                     if (!$serverInfo) {
                         if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
                             if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
                                 continue;
                             } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
                                 $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                                 Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId));
                                 continue;
                             }
                             if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
                                 $action = 'terminate';
                                 if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
                                     $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
                                 }
                                 if ($action == 'flag') {
                                     $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
                                 } else {
                                     $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                                     \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
                                 }
                             } else {
                                 $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
                                 Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
                             }
                             continue;
                         }
                     } else {
                         //http.persistent.handles.limit must be set to 0 for pecl-http version 1
                         $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
                         //More debug
                         $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache)));
                         $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}"));
                     }
                 } else {
                     $DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]);
                 }
             }
         } catch (Exception $e) {
             if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
                 /* @var $env \Scalr_Environment */
                 $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
                 $env->status = Scalr_Environment::STATUS_INACTIVE;
                 $env->save();
                 //Saving the reason why this environment is disabled
                 $env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]);
                 return;
             } elseif (stristr($e->getMessage(), "Could not connect to host")) {
                 continue;
             }
             $this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine());
             continue;
         }
         try {
             if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
                 $openstackErrorState = false;
                 if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
                     $openstackErrorState = true;
                 }
                 if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
                     if ($openstackErrorState) {
                         try {
                             $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
                             Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId));
                         } catch (Exception $e) {
                         }
                     } else {
                         Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
                     }
                     $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                     $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
                     \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
                     continue;
                 } elseif ($DBServer->GetRealStatus()->isSuspended()) {
                     //In case the server was suspended when it was running
                     if ($DBServer->status == SERVER_STATUS::RUNNING) {
                         Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
                         $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
                         $DBServer->remoteIp = "";
                         $DBServer->localIp = "";
                         $DBServer->status = SERVER_STATUS::SUSPENDED;
                         $DBServer->Save();
                         $event = new HostDownEvent($DBServer);
                         $event->isSuspended = true;
                         \Scalr::FireEvent($DBFarm->ID, $event);
                         continue;
                     } else {
                         //If the server was suspended during initialization
                         //we do not support this and need to terminate this instance
                         $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                         continue;
                     }
                 }
             }
             if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
                 if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
                     $platform = PlatformFactory::NewPlatform($DBServer->platform);
                     if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
                         if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
                             continue;
                         }
                     }
                     // For Openstack we need to re-accociate IPs
                     try {
                         if ($DBServer->isOpenstack()) {
                             $this->openstackSetFloatingIp($DBServer);
                         }
                     } catch (Exception $e) {
                         if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
                             $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
                         }
                     }
                     if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
                         if (!$DBServer->remoteIp) {
                             $DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer);
                             $DBServer->Save();
                         }
                     }
                     if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) {
                         $DBServer->status = \SERVER_STATUS::PENDING;
                         $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1);
                         $DBServer->dateAdded = date("Y-m-d H:i:s");
                         $DBServer->Save();
                     } else {
                         $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
                         \Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
                     }
                     continue;
                 } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
                     if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
                         if ($DBServer->status == SERVER_STATUS::PENDING) {
                             if (!$DBServer->remoteIp && !$DBServer->localIp) {
                                 $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
                                 if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) {
                                     $DBServer->remoteIp = $ipaddresses['remoteIp'];
                                     $DBServer->localIp = $ipaddresses['localIp'];
                                     $DBServer->Save();
                                 }
                                 //Add tags
                                 Ec2Helper::createServerTags($DBServer);
                             }
                             if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
                                 if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
                                     $ipAddress = Ec2EipHelper::setEipForServer($DBServer);
                                     if ($ipAddress) {
                                         $DBServer->remoteIp = $ipAddress;
                                         $DBServer->Save();
                                     }
                                 }
                             }
                         }
                     }
                     try {
                         if ($DBServer->isOpenstack()) {
                             $this->openstackSetFloatingIp($DBServer);
                         }
                     } catch (Exception $e) {
                         if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
                             $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
                         }
                     }
                     if ($DBServer->isCloudstack()) {
                         if ($DBServer->status == SERVER_STATUS::PENDING) {
                             $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
                             try {
                                 $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
                                 $res = $cs->queryAsyncJobResult($jobId);
                                 if ($res->jobstatus == 1) {
                                     $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
                                 }
                                 //TODO handle failed job: $res->jobresult->jobstatus == 2
                             } catch (Exception $e) {
                                 if ($DBServer->farmId) {
                                     Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
                                 }
                             }
                         }
                     }
                     try {
                         $dtadded = strtotime($DBServer->dateAdded);
                         $DBFarmRole = $DBServer->GetFarmRoleObject();
                         $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
                     } catch (Exception $e) {
                         if (stristr($e->getMessage(), "not found")) {
                             $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
                         }
                     }
                     $scripting_event = false;
                     if ($DBServer->status == SERVER_STATUS::PENDING) {
                         $event = "hostInit";
                         $scripting_event = EVENT_TYPE::HOST_INIT;
                     } elseif ($DBServer->status == SERVER_STATUS::INIT) {
                         $event = "hostUp";
                         $scripting_event = EVENT_TYPE::HOST_UP;
                     }
                     if ($scripting_event && $dtadded) {
                         $scripting_timeout = (int) $db->GetOne("\n                                SELECT SUM(timeout)\n                                FROM farm_role_scripts\n                                WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n                            ", [$scripting_event, $DBServer->farmRoleId]);
                         if ($scripting_timeout) {
                             $launch_timeout = $launch_timeout + $scripting_timeout;
                         }
                         if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
                             //Add entry to farm log
                             Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId));
                             try {
                                 $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
                             } catch (Exception $err) {
                                 $this->getLogger()->fatal($err->getMessage());
                             }
                         } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
                             //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
                             //THINK ABOUT WORKAROUND
                         }
                     }
                     //Whether IP address is changed
                     if (!$DBServer->IsRebooting()) {
                         $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
                         if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
                             Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
                             \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
                         }
                         //TODO Check health
                     }
                 }
             } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
                 if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
                     $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                     \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
                 } elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) {
                     \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
                 }
             } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
                 // Is IP address changed?
                 if (!$DBServer->IsRebooting()) {
                     $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
                     if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
                         \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
                     }
                     if ($payAsYouGoTime) {
                         $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
                         if ($initTime < $payAsYouGoTime) {
                             $initTime = $payAsYouGoTime;
                         }
                         $runningHours = ceil((time() - $initTime) / 3600);
                         $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
                         $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
                     }
                     if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
                         $env = Scalr_Environment::init()->loadById($DBServer->envId);
                         $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
                         $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
                         if (!$time || time() < $time + 1200) {
                             $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
                             $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
                         }
                     }
                 } else {
                     //TODO Check reboot timeout
                 }
             }
         } catch (Exception $e) {
             if (stristr($e->getMessage(), "not found")) {
                 $this->getLogger()->fatal($e->getMessage());
             } elseif (stristr($e->getMessage(), "Request limit exceeded")) {
                 sleep(5);
                 $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
             } else {
                 $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
             }
         }
     }
     return $request;
 }
Example #7
0
 /**
  * {@inheritdoc}
  * @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
  */
 public function worker($request)
 {
     $db = \Scalr::getDb();
     //The list of the suspension information about cloud platforms
     $this->aSuspensionInfo = [];
     //Speed up poller
     if ($this->config()->daemon) {
         //Warming up static DI cache
         \Scalr::getContainer()->warmup();
     }
     // Reconfigure observers
     \Scalr::ReconfigureObservers();
     $DBFarm = DBFarm::LoadByID($request->farmId);
     $account = Scalr_Account::init()->loadById($DBFarm->ClientID);
     $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
     $transactionId = abs(crc32(posix_getpid() . $request->farmId));
     $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform);
     $jobStartTime = microtime(true);
     //Retrieves the number of either terminated or suspended servers for the farm
     $servers_count = $db->GetOne("\n            SELECT COUNT(*) AS cnt FROM servers\n            WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n        ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
     if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
         //There are no servers for this farm and platform
         return;
     }
     $this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform);
     $config = \Scalr::getContainer()->config;
     /*
     if ($request->platform) {
         $p = PlatformFactory::NewPlatform($request->platform);
         $p->ClearCache();
     }
     */
     $p = PlatformFactory::NewPlatform($request->platform);
     foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) {
         /* @var $DBServer \DBServer */
         //Get platform suspension info
         $suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId);
         //If the cloud platform is suspended we should not process it
         if ($suspensionInfo->isSuspended()) {
             continue;
         }
         try {
             //1. We need to check that server is exists in cloud and not missed.
             //   (On Openstack server can be missed and should not be terminated)
             $cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation);
             if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) {
                 try {
                     $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform);
                     if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
                         //For Azure we need to pass resource group instead of cloudLocation
                         $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP));
                     } else {
                         $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
                     }
                     //We successfully polled cloud so can resume suspension status for the cloud platform
                     if ($suspensionInfo->isPendingSuspend()) {
                         $suspensionInfo->resume();
                     }
                 } catch (Exception $e) {
                     if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
                         $suspensionInfo->registerError($e->getMessage());
                     }
                     $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
                     continue;
                 }
             }
             if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
                 if (!$p->IsServerExists($DBServer)) {
                     try {
                         $serverInfo = $p->GetServerExtendedInformation($DBServer);
                     } catch (Exception $e) {
                         $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
                         continue;
                     }
                     if (!$serverInfo) {
                         if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
                             if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
                                 continue;
                             } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
                                 $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                                 \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId));
                                 continue;
                             }
                             $action = 'terminate';
                             if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
                                 $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
                             }
                             if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) {
                                 \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
                                 $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]);
                             } else {
                                 \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
                                 $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                             }
                             continue;
                         }
                     } else {
                         //http.persistent.handles.limit must be set to 0 for pecl-http version 1
                         $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
                     }
                 } else {
                     $DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]);
                 }
             }
         } catch (Exception $e) {
             if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
                 $suspensionInfo->registerError($e->getMessage());
             }
             $this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine());
             continue;
         }
         try {
             if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
                 $openstackErrorState = false;
                 if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
                     $openstackErrorState = true;
                 }
                 if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
                     // If openstack server is in ERROR state we need more details
                     if ($openstackErrorState) {
                         try {
                             $info = $p->GetServerExtendedInformation($DBServer);
                             $status = empty($info['Status']) ? false : $info['Status'];
                         } catch (Exception $e) {
                         }
                     }
                     if (empty($status)) {
                         $status = $DBServer->GetRealStatus()->getName();
                     }
                     \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId));
                     $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                     continue;
                 } elseif ($DBServer->GetRealStatus()->isSuspended()) {
                     //In case the server was suspended when it was running
                     if ($DBServer->status == SERVER_STATUS::RUNNING) {
                         \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
                         $event = new HostDownEvent($DBServer);
                         $event->isSuspended = true;
                         \Scalr::FireEvent($DBFarm->ID, $event);
                         continue;
                     } else {
                         if ($DBServer->status != \SERVER_STATUS::RESUMING) {
                             //If the server was suspended during initialization
                             //we do not support this and need to terminate this instance
                             if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
                                 try {
                                     $info = $p->GetServerExtendedInformation($DBServer);
                                     $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
                                 } catch (\Exception $e) {
                                     // no need to do anything here;
                                 }
                                 $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
                             }
                             $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
                             continue;
                         } else {
                             // Need to clear cache, because this situation happens only when cache is stale.
                             $p->ClearCache();
                         }
                     }
                 }
             }
             if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
                 if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
                     if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
                         if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
                             continue;
                         }
                     }
                     $update = [];
                     // For Openstack we need to re-accociate IPs
                     try {
                         if ($DBServer->isOpenstack()) {
                             OpenstackHelper::setServerFloatingIp($DBServer);
                         }
                     } catch (Exception $e) {
                         if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
                             $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
                         }
                     }
                     if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
                         if (!$DBServer->remoteIp) {
                             $update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer);
                         }
                     }
                     if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
                         try {
                             $info = $p->GetServerExtendedInformation($DBServer);
                             $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
                         } catch (\Exception $e) {
                             // no need to do anything here;
                         }
                         $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
                     }
                     $update['status'] = \SERVER_STATUS::RESUMING;
                     $update['dateAdded'] = date("Y-m-d H:i:s");
                     $DBServer->update($update);
                     unset($update);
                     continue;
                 } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) {
                     $elasticIpAssigned = false;
                     if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
                         if ($DBServer->status == SERVER_STATUS::PENDING) {
                             if (!$DBServer->remoteIp && !$DBServer->localIp) {
                                 $ipaddresses = $p->GetServerIPAddresses($DBServer);
                                 $elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer);
                                 if ($elasticIpAddress) {
                                     $ipaddresses['remoteIp'] = $elasticIpAddress;
                                     $DBServer->remoteIp = $elasticIpAddress;
                                     $elasticIpAssigned = true;
                                 }
                                 if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) {
                                     $DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]);
                                 }
                                 //Add tags
                                 Ec2Helper::createObjectTags($DBServer);
                             }
                         }
                     }
                     if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
                         if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) {
                             if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
                                 // Check scalarizr deployment status
                                 $env = $DBServer->GetEnvironmentObject();
                                 $azure = $env->azure();
                                 $info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME));
                                 $extensions = !empty($info->extensions) ? $info->extensions : [];
                                 foreach ($extensions as $extension) {
                                     /* @var $extension ExtensionData */
                                     if ($extension->name == 'scalarizr') {
                                         $extStatus = $extension->statuses[0];
                                         /* @var $extStatus StatusData */
                                         if ($extStatus->level == 'Error') {
                                             $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]);
                                         }
                                     }
                                 }
                             }
                         } else {
                             AzureHelper::setupScalrAgent($DBServer);
                         }
                     }
                     try {
                         if ($DBServer->isOpenstack()) {
                             OpenstackHelper::setServerFloatingIp($DBServer);
                         }
                     } catch (Exception $e) {
                         if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
                             $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
                         }
                     }
                     if ($DBServer->isCloudstack()) {
                         if ($DBServer->status == SERVER_STATUS::PENDING) {
                             $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
                             try {
                                 $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
                                 $res = $cs->queryAsyncJobResult($jobId);
                                 if ($res->jobstatus == 1) {
                                     $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
                                 }
                                 //TODO handle failed job: $res->jobresult->jobstatus == 2
                             } catch (Exception $e) {
                                 if ($DBServer->farmId) {
                                     \Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
                                 }
                             }
                         }
                     }
                     try {
                         $dtadded = strtotime($DBServer->dateAdded);
                         $DBFarmRole = $DBServer->GetFarmRoleObject();
                         $launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900;
                     } catch (Exception $e) {
                         if (stristr($e->getMessage(), "not found")) {
                             $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
                         }
                     }
                     $scriptingEvent = false;
                     $eventName = null;
                     if ($DBServer->status == SERVER_STATUS::PENDING) {
                         $eventName = "hostInit";
                         $scriptingEvent = EVENT_TYPE::HOST_INIT;
                     } elseif ($DBServer->status == SERVER_STATUS::INIT) {
                         $eventName = "hostUp";
                         $scriptingEvent = EVENT_TYPE::HOST_UP;
                     }
                     if ($scriptingEvent && $dtadded) {
                         $hasPendingMessages = !!$db->GetOne("\n                                SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n                            ", [$DBServer->serverId]);
                         $scriptingTimeout = (int) $db->GetOne("\n                                SELECT SUM(timeout)\n                                FROM farm_role_scripts\n                                WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n                            ", [$scriptingEvent, $DBServer->farmRoleId]);
                         if ($scriptingTimeout) {
                             $launchTimeout = $launchTimeout + $scriptingTimeout;
                         }
                         if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
                             //Add entry to farm log
                             \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId));
                             try {
                                 $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false);
                             } catch (Exception $err) {
                                 $this->getLogger()->fatal($err->getMessage());
                             }
                         } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
                             //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
                             //THINK ABOUT WORKAROUND
                         }
                     }
                     //Whether IP address is changed
                     if (!$DBServer->IsRebooting() && !$elasticIpAssigned) {
                         $ipaddresses = $p->GetServerIPAddresses($DBServer);
                         if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
                             \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
                             \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
                         }
                         //TODO Check health
                     }
                 }
             } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
                 //TODO: Terminated outside scalr while in SUSPENDED state
                 $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
             } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
                 // Is IP address changed?
                 if (!$DBServer->IsRebooting()) {
                     $ipaddresses = $p->GetServerIPAddresses($DBServer);
                     // Private IP cannot be removed (only changed).
                     if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
                         \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
                     }
                     if ($payAsYouGoTime) {
                         $initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null;
                         if ($initTime < $payAsYouGoTime) {
                             $initTime = $payAsYouGoTime;
                         }
                         $runningHours = ceil((time() - $initTime) / 3600);
                         $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform);
                         $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
                     }
                     if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
                         $env = Scalr_Environment::init()->loadById($DBServer->envId);
                         $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
                         $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
                         if (!$time || time() > $time + 1200) {
                             $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
                             $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
                         }
                     }
                 } else {
                     //TODO Check reboot timeout
                 }
             }
         } catch (Exception $e) {
             if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
                 $suspensionInfo->registerError($e->getMessage());
             }
             if (stristr($e->getMessage(), "not found")) {
                 $this->getLogger()->fatal($e->getMessage());
             } elseif (stristr($e->getMessage(), "Request limit exceeded")) {
                 sleep(5);
                 $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
             } else {
                 $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
             }
         }
     }
     $this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime);
     return $request;
 }
Example #8
0
 /**
  * Creates clone for the farm
  *
  * @param   string             $name   The name of the farm
  * @param   Scalr_Account_User $user   The user object
  * @param   int                $envId  The identifier of the environment
  * @return  DBFarm             Returns clone
  */
 public function cloneFarm($name = false, Scalr_Account_User $user, $envId)
 {
     $account = $user->getAccount();
     $account->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
     $definition = $this->getDefinition();
     if (!$name) {
         $template = "";
         if (!preg_match('/^(.*?)\\(clone \\#([0-9]*)\\)$/si', $definition->name)) {
             $name = $definition->name;
         } else {
             preg_match('/^(.*?)\\(clone \\#([0-9]*)\\)$/si', $definition->name, $matches);
             $name = trim($matches[1]);
         }
         $name = preg_replace('/[^A-Za-z0-9_\\. -]+/', '', $name);
         $lastUsedIndex = $this->DB->GetOne('SELECT MAX(CAST((SUBSTR(@p:=SUBSTRING_INDEX(name, \'#\', -1), 1, LENGTH(@p) - 1)) AS UNSIGNED)) as lastUsedCloneNumber FROM farms WHERE name REGEXP \'' . str_replace('.', '\\.', $name) . ' \\\\(clone #[0-9]+\\\\)\' AND env_id = ?', array($envId));
         $name = $name . ' (clone #' . ($lastUsedIndex + 1) . ')';
     }
     $dbFarm = self::create($name, $user, $envId);
     $dbFarm->createdByUserId = $user->id;
     $dbFarm->createdByUserEmail = $user->getEmail();
     $dbFarm->RolesLaunchOrder = $definition->rolesLaunchOrder;
     $dbFarm->teamId = $definition->teamId;
     $dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $definition->settings[DBFarm::SETTING_TIMEZONE]);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $definition->settings[DBFarm::SETTING_EC2_VPC_ID]);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $definition->settings[DBFarm::SETTING_EC2_VPC_REGION]);
     $dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_REPOSITORY, $definition->settings[DBFarm::SETTING_SZR_UPD_REPOSITORY]);
     $dbFarm->SetSetting(DBFarm::SETTING_SZR_UPD_SCHEDULE, $definition->settings[DBFarm::SETTING_SZR_UPD_SCHEDULE]);
     $dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, $definition->settings[DBFarm::SETTING_LEASE_STATUS]);
     $variables = new Scalr_Scripting_GlobalVariables($dbFarm->ClientID, $envId, Scalr_Scripting_GlobalVariables::SCOPE_FARM);
     $variables->setValues($definition->globalVariables, 0, $dbFarm->ID, 0);
     foreach ($definition->roles as $index => $role) {
         $dbFarmRole = $dbFarm->AddRole(DBRole::loadById($role->roleId), $role->platform, $role->cloudLocation, $index + 1, $role->alias);
         $oldRoleSettings = $dbFarmRole->GetAllSettings();
         $dbFarmRole->applyDefinition($role, true);
         $newSettings = $dbFarmRole->GetAllSettings();
         // Platform specified updates
         if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
             \Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
             \Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
         if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
             CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $newSettings);
         }
         $dbFarmRolesList[] = $dbFarmRole;
         $usedPlatforms[$role->platform] = 1;
     }
     $dbFarm->save();
     return $dbFarm;
 }
Example #9
0
 public function xBuildAction()
 {
     $this->request->defineParams(array('farmId' => array('type' => 'int'), 'roles' => array('type' => 'json'), 'farm' => array('type' => 'json'), 'roleUpdate' => array('type' => 'int')));
     $this->request->restrictAccess(Acl::RESOURCE_FARMS, Acl::PERM_FARMS_MANAGE);
     if (!$this->isFarmConfigurationValid($this->getParam('farmId'), $this->getParam('farm'), (array) $this->getParam('roles'))) {
         if ($this->errors['error_count'] != 0) {
             $this->response->failure();
             $this->response->data(array('errors' => $this->errors));
             return;
         }
     }
     $farm = $this->getParam('farm');
     $client = Client::Load($this->user->getAccountId());
     if ($this->getParam('farmId')) {
         $dbFarm = DBFarm::LoadByID($this->getParam('farmId'));
         $this->user->getPermissions()->validate($dbFarm);
         $dbFarm->isLocked();
         if ($this->getParam('changed') && $dbFarm->changedTime && $this->getParam('changed') != $dbFarm->changedTime) {
             $userName = '******';
             $changed = explode(' ', $this->getParam('changed'));
             $changedTime = intval($changed[1]);
             try {
                 $user = new Scalr_Account_User();
                 $user->loadById($dbFarm->changedByUserId);
                 $userName = $user->getEmail();
             } catch (Exception $e) {
             }
             $this->response->failure();
             $this->response->data(array('changedFailure' => sprintf('%s changed this farm at %s', $userName, Scalr_Util_DateTime::convertTz($changedTime))));
             return;
         }
         $dbFarm->changedByUserId = $this->user->getId();
         $dbFarm->changedTime = microtime();
         $bNew = false;
     } else {
         $this->user->getAccount()->validateLimit(Scalr_Limits::ACCOUNT_FARMS, 1);
         $dbFarm = new DBFarm();
         $dbFarm->ClientID = $this->user->getAccountId();
         $dbFarm->EnvID = $this->getEnvironmentId();
         $dbFarm->Status = FARM_STATUS::TERMINATED;
         $dbFarm->createdByUserId = $this->user->getId();
         $dbFarm->createdByUserEmail = $this->user->getEmail();
         $dbFarm->changedByUserId = $this->user->getId();
         $dbFarm->changedTime = microtime();
         $bNew = true;
     }
     if ($this->getParam('farm')) {
         $dbFarm->Name = $this->request->stripValue($farm['name']);
         $dbFarm->RolesLaunchOrder = $farm['rolesLaunchOrder'];
         $dbFarm->Comments = $this->request->stripValue($farm['description']);
     }
     if (empty($dbFarm->Name)) {
         throw new Exception(_("Farm name required"));
     }
     if (!$bNew && $farm['owner'] && $farm['owner'] != $dbFarm->createdByUserId) {
         if ($dbFarm->createdByUserId == $this->user->getId() || $this->user->isAccountOwner()) {
             $user = (new Scalr_Account_User())->loadById($farm['owner']);
             $dbFarm->createdByUserId = $user->getId();
             $dbFarm->createdByUserEmail = $user->getEmail();
             // TODO: move to subclass \Farm\Setting\OwnerHistory
             $history = unserialize($dbFarm->GetSetting(DBFarm::SETTING_OWNER_HISTORY));
             if (!is_array($history)) {
                 $history = [];
             }
             $history[] = ['newId' => $user->getId(), 'newEmail' => $user->getEmail(), 'changedById' => $this->user->getId(), 'changedByEmail' => $this->user->getEmail(), 'dt' => date('Y-m-d H:i:s')];
             $dbFarm->SetSetting(DBFarm::SETTING_OWNER_HISTORY, serialize($history));
         }
     }
     $dbFarm->save();
     $governance = new Scalr_Governance($this->getEnvironmentId());
     if (!$this->getParam('farmId') && $governance->isEnabled(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE)) {
         $dbFarm->SetSetting(DBFarm::SETTING_LEASE_STATUS, 'Active');
         // for created farm
     }
     if (isset($farm['variables'])) {
         $variables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), Scalr_Scripting_GlobalVariables::SCOPE_FARM);
         $variables->setValues(is_array($farm['variables']) ? $farm['variables'] : [], 0, $dbFarm->ID, 0, '', false, true);
     }
     if (!$farm['timezone']) {
         $farm['timezone'] = date_default_timezone_get();
     }
     $dbFarm->SetSetting(DBFarm::SETTING_TIMEZONE, $farm['timezone']);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_ID, $farm['vpc_id']);
     $dbFarm->SetSetting(DBFarm::SETTING_EC2_VPC_REGION, $farm['vpc_region']);
     if (!$dbFarm->GetSetting(DBFarm::SETTING_CRYPTO_KEY)) {
         $dbFarm->SetSetting(DBFarm::SETTING_CRYPTO_KEY, Scalr::GenerateRandomKey(40));
     }
     if ($this->getContainer()->analytics->enabled) {
         if ($this->request->isInterfaceBetaOrNotHostedScalr()) {
             //Cost analytics project must be set for the Farm object
             $dbFarm->setProject(!empty($farm['projectId']) ? $farm['projectId'] : null, $this->request->isInterfaceBetaOrNotHostedScalr());
         } else {
             if (isset($bNew)) {
                 //Default project is set for hosted scalr accounts. Users cannot manage it.
                 $farm['projectId'] = $dbFarm->setProject(null);
             }
         }
     }
     $virtualFarmRoles = array();
     $roles = $this->getParam('roles');
     if (!empty($roles)) {
         foreach ($roles as $role) {
             if (strpos($role['farm_role_id'], "virtual_") !== false) {
                 $dbRole = DBRole::loadById($role['role_id']);
                 $dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index'], $role['alias']);
                 $virtualFarmRoles[$role['farm_role_id']] = $dbFarmRole->ID;
             }
         }
     }
     $usedPlatforms = array();
     $dbFarmRolesList = array();
     $newFarmRolesList = array();
     $farmRoleVariables = new Scalr_Scripting_GlobalVariables($this->user->getAccountId(), $this->getEnvironmentId(), Scalr_Scripting_GlobalVariables::SCOPE_FARMROLE);
     if (!empty($roles)) {
         foreach ($roles as $role) {
             if ($role['farm_role_id']) {
                 if ($virtualFarmRoles[$role['farm_role_id']]) {
                     $role['farm_role_id'] = $virtualFarmRoles[$role['farm_role_id']];
                 }
                 $update = true;
                 $dbFarmRole = DBFarmRole::LoadByID($role['farm_role_id']);
                 $dbRole = DBRole::loadById($dbFarmRole->RoleID);
                 $role['role_id'] = $dbFarmRole->RoleID;
                 if ($dbFarmRole->Platform == SERVER_PLATFORMS::GCE) {
                     $dbFarmRole->CloudLocation = $role['cloud_location'];
                 }
             } else {
                 $update = false;
                 $dbRole = DBRole::loadById($role['role_id']);
                 $dbFarmRole = $dbFarm->AddRole($dbRole, $role['platform'], $role['cloud_location'], (int) $role['launch_index']);
             }
             if ($dbRole->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
                 $role['settings'][DBFarmRole::SETTING_SCALING_MAX_INSTANCES] = $role['settings'][DBFarmRole::SETTING_SCALING_MIN_INSTANCES];
             }
             if ($dbFarmRole->NewRoleID) {
                 continue;
             }
             if ($update) {
                 $dbFarmRole->LaunchIndex = (int) $role['launch_index'];
                 $dbFarmRole->Alias = $role['alias'];
                 $dbFarmRole->Save();
             }
             $usedPlatforms[$role['platform']] = 1;
             $oldRoleSettings = $dbFarmRole->GetAllSettings();
             // Update virtual farm_role_id with actual value
             $scripts = (array) $role['scripting'];
             if (count($virtualFarmRoles) > 0) {
                 array_walk_recursive($scripts, function (&$v, $k) use($virtualFarmRoles) {
                     if (is_string($v)) {
                         $v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
                     }
                 });
                 array_walk_recursive($role['settings'], function (&$v, $k) use($virtualFarmRoles) {
                     if (is_string($v)) {
                         $v = str_replace(array_keys($virtualFarmRoles), array_values($virtualFarmRoles), $v);
                     }
                 });
             }
             $dbFarmRole->ClearSettings("chef.");
             if (!empty($role['scaling_settings']) && is_array($role['scaling_settings'])) {
                 foreach ($role['scaling_settings'] as $k => $v) {
                     $dbFarmRole->SetSetting($k, $v, DBFarmRole::TYPE_CFG);
                 }
             }
             foreach ($role['settings'] as $k => $v) {
                 $dbFarmRole->SetSetting($k, $v, DBFarmRole::TYPE_CFG);
             }
             /****** Scaling settings ******/
             $scalingManager = new Scalr_Scaling_Manager($dbFarmRole);
             $scalingManager->setFarmRoleMetrics(is_array($role['scaling']) ? $role['scaling'] : array());
             //TODO: optimize this code...
             $this->db->Execute("DELETE FROM farm_role_scaling_times WHERE farm_roleid=?", array($dbFarmRole->ID));
             // 5 = Time based scaling -> move to constants
             if ($role['scaling'][5]) {
                 foreach ($role['scaling'][5] as $scal_period) {
                     $chunks = explode(":", $scal_period['id']);
                     $this->db->Execute("INSERT INTO farm_role_scaling_times SET\n                            farm_roleid\t\t= ?,\n                            start_time\t\t= ?,\n                            end_time\t\t= ?,\n                            days_of_week\t= ?,\n                            instances_count\t= ?\n                        ", array($dbFarmRole->ID, $chunks[0], $chunks[1], $chunks[2], $chunks[3]));
                 }
             }
             /*****************/
             /* Update role params */
             $dbFarmRole->SetParameters((array) $role['params']);
             /* End of role params management */
             /* Add script options to databse */
             $dbFarmRole->SetScripts($scripts, (array) $role['scripting_params']);
             /* End of scripting section */
             /* Add services configuration */
             $dbFarmRole->SetServiceConfigPresets((array) $role['config_presets']);
             /* End of scripting section */
             /* Add storage configuration */
             if (isset($role['storages'])) {
                 if (isset($role['storages']['configs'])) {
                     $dbFarmRole->getStorage()->setConfigs($role['storages']['configs']);
                 }
             }
             $farmRoleVariables->setValues(is_array($role['variables']) ? $role['variables'] : [], $dbFarmRole->GetRoleID(), $dbFarm->ID, $dbFarmRole->ID, '', false, true);
             Scalr_Helpers_Dns::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
             foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
                 $behavior->onFarmSave($dbFarm, $dbFarmRole);
             }
             /**
              * Platfrom specified updates
              */
             if ($dbFarmRole->Platform == SERVER_PLATFORMS::EC2) {
                 \Scalr\Modules\Platforms\Ec2\Helpers\EbsHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
                 \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
                 \Scalr\Modules\Platforms\Ec2\Helpers\ElbHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
             }
             if (in_array($dbFarmRole->Platform, array(SERVER_PLATFORMS::IDCF, SERVER_PLATFORMS::CLOUDSTACK))) {
                 Scalr\Modules\Platforms\Cloudstack\Helpers\CloudstackHelper::farmUpdateRoleSettings($dbFarmRole, $oldRoleSettings, $role['settings']);
             }
             $dbFarmRolesList[] = $dbFarmRole;
             $newFarmRolesList[] = $dbFarmRole->ID;
         }
     }
     if (!$this->getParam('roleUpdate')) {
         foreach ($dbFarm->GetFarmRoles() as $dbFarmRole) {
             if (!$dbFarmRole->NewRoleID && !in_array($dbFarmRole->ID, $newFarmRolesList)) {
                 $dbFarmRole->Delete();
             }
         }
     }
     if ($usedPlatforms[SERVER_PLATFORMS::CLOUDSTACK]) {
         \Scalr\Modules\Platforms\Cloudstack\Helpers\CloudstackHelper::farmSave($dbFarm, $dbFarmRolesList);
     }
     if ($usedPlatforms[SERVER_PLATFORMS::EC2]) {
         \Scalr\Modules\Platforms\Ec2\Helpers\Ec2Helper::farmSave($dbFarm, $dbFarmRolesList);
     }
     if ($usedPlatforms[SERVER_PLATFORMS::EUCALYPTUS]) {
         \Scalr\Modules\Platforms\Eucalyptus\Helpers\EucalyptusHelper::farmSave($dbFarm, $dbFarmRolesList);
     }
     $dbFarm->save();
     if (!$client->GetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED)) {
         $client->SetSettingValue(CLIENT_SETTINGS::DATE_FARM_CREATED, time());
     }
     $this->response->success('Farm successfully saved');
     $this->response->data(array('farmId' => $dbFarm->ID));
 }