/** * {@inheritdoc} * @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::startChild() */ function startChild() { // Reopen DB connection in child $this->db = $this->getContainer()->adodb; // Reconfigure observers; Scalr::ReconfigureObservers(); }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); try { $dbFarm = DBFarm::LoadByID($request->farmId); $curDate = new DateTime(); $tdValue = $dbFarm->GetSetting(DBFarm::SETTING_LEASE_TERMINATE_DATE); if ($tdValue) { $td = new DateTime($tdValue); if ($td < $curDate) { //Terminates farm $event = new FarmTerminatedEvent(0, 1, false, 1); SettingEntity::increase(SettingEntity::LEASE_TERMINATE_FARM); \Scalr::FireEvent($request->farmId, $event); $this->log('INFO', sprintf('Farm: %s [ID: %d] was terminated by lease manager', $dbFarm->Name, $dbFarm->ID)); } else { // only inform user $days = $td->diff($curDate)->days; $notifications = json_decode($dbFarm->GetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND), true); $governance = new Scalr_Governance($dbFarm->EnvID); $settings = $governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'notifications'); if (is_array($settings)) { foreach ($settings as $n) { if (!$notifications[$n['key']] && $n['period'] >= $days) { $mailer = \Scalr::getContainer()->mailer; $tdHuman = Scalr_Util_DateTime::convertDateTime($td, $dbFarm->GetSetting(DBFarm::SETTING_TIMEZONE), 'M j, Y'); if ($n['to'] == 'owner') { $user = new Scalr_Account_User(); $user->loadById($dbFarm->createdByUserId); if (\Scalr::config('scalr.auth_mode') == 'ldap') { $email = $user->getSetting(Scalr_Account_User::SETTING_LDAP_EMAIL); if (!$email) { $email = $user->getEmail(); } } else { $email = $user->getEmail(); } $mailer->addTo($email); } else { foreach (explode(',', $n['emails']) as $email) { $mailer->addTo(trim($email)); } } $mailer->sendTemplate(SCALR_TEMPLATES_PATH . '/emails/farm_lease_terminate.eml', array('{{terminate_date}}' => $tdHuman, '{{farm}}' => $dbFarm->Name, '{{envName}}' => $dbFarm->GetEnvironmentObject()->name, '{{envId}}' => $dbFarm->GetEnvironmentObject()->id)); $notifications[$n['key']] = 1; $dbFarm->SetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND, json_encode($notifications)); $this->log('INFO', "Notification was sent by key: %s about farm: %s [ID: %d] by lease manager", $n['key'], $dbFarm->Name, $dbFarm->ID); } } } } } } catch (Exception $e) { throw $e; } return $request; }
function startChild() { // Reopen DB connection in child $this->db = Core::GetDBInstance(null, true); // Reconfigure observers; Scalr::ReconfigureObservers(); }
function startChild() { // Reopen DB connection in child $this->db = Core::GetDBInstance(null, true); // Reconfigure observers; Scalr::ReconfigureObservers(); $this->snmpWatcher = new SNMPWatcher(); foreach (array_keys($this->watchers) as $watcher_name) { $this->snmpWatcher->SetOIDs($watcher_name); } }
public function StartThread($data) { // Reconfigure observers; Scalr::ReconfigureObservers(); // // Create pid file // @file_put_contents(CACHEPATH . "/" . __CLASS__ . ".Daemon.pid", posix_getpid()); // Get memory usage on start $memory_usage = $this->GetMemoryUsage(); $this->Logger->info("SzrMessaging daemon started. Memory usage: {$memory_usage}M"); // Get DB instance $db = Core::GetDBInstance(); $FarmObservers = array(); $job = new Scalr_Cronjob_ScalarizrMessaging(); while (true) { $rows = $db->Execute("SELECT distinct(server_id) FROM messages WHERE type = ? AND status = ? AND isszr = ?", array("in", MESSAGE_STATUS::PENDING, 1)); while ($row = $rows->FetchRow()) { try { $serverId = $row['server_id']; $job->handleWork($serverId); } catch (Exception $e) { } } // Cleaning unset($current_memory_usage); unset($event); // Check memory usage $current_memory_usage = $this->GetMemoryUsage() - $memory_usage; if ($current_memory_usage > $this->DaemonMemoryLimit) { $this->Logger->warn("SzrMessaging daemon reached memory limit {$this->DaemonMemoryLimit}M, Used:{$current_memory_usage}M"); $this->Logger->warn("Restart daemon."); exit; } // Sleep for 60 seconds sleep(5); // Clear stat file cache clearstatcache(); // Check daemon file for modifications. if ($this->DaemonMtime && $this->DaemonMtime < @filemtime(__FILE__)) { $this->Logger->warn(__FILE__ . " - updated. Exiting for daemon reload."); exit; } } }
public function StartThread($data) { // Reconfigure observers; Scalr::ReconfigureObservers(); // // Create pid file // @file_put_contents(CACHEPATH . "/" . __CLASS__ . ".Daemon.pid", posix_getpid()); // Get memory usage on start $memory_usage = $this->GetMemoryUsage(); $this->Logger->info("DBQueueEventProcess daemon started. Memory usage: {$memory_usage}M"); // Get DB instance $db = Core::GetDBInstance(); $FarmObservers = array(); while (true) { // Process tasks from Deferred event queue while ($Task = TaskQueue::Attach(QUEUE_NAME::DEFERRED_EVENTS)->Poll()) { $Task->Run(); } // Reset task TaskQueue::Attach(QUEUE_NAME::DEFERRED_EVENTS)->Reset(); // Cleaning unset($current_memory_usage); unset($event); // Check memory usage $current_memory_usage = $this->GetMemoryUsage() - $memory_usage; if ($current_memory_usage > $this->DaemonMemoryLimit) { $this->Logger->warn("DBQueueEventProcess daemon reached memory limit {$this->DaemonMemoryLimit}M, Used:{$current_memory_usage}M"); $this->Logger->warn("Restart daemon."); exit; } // Sleep for 60 seconds sleep(15); // Clear stat file cache clearstatcache(); // Check daemon file for modifications. if ($this->DaemonMtime && $this->DaemonMtime < @filemtime(__FILE__)) { $this->Logger->warn(__FILE__ . " - updated. Exiting for daemon reload."); exit; } } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $dtNow = new DateTime('now'); try { $dbServer = DBServer::LoadByID($request->serverId); } catch (ServerNotFoundException $e) { $this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage()); return false; } if (!in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED])) { return false; } // Check and skip locked instances if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) { if (($checkDateTime = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME)) <= time()) { if (!$dbServer->GetRealStatus(true)->isTerminated()) { $isLocked = $dbServer->GetEnvironmentObject()->aws($dbServer->GetCloudLocation())->ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); if ($isLocked) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Server '%s' has disableAPITermination flag and can't be terminated (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); $startTime = strtotime($dbServer->dateShutdownScheduled); // 1, 2, 3, 4, 5, 6, 9, 14, ... 60 $diff = round((($checkDateTime < $startTime ? $startTime : $checkDateTime) - $startTime) / 60 * 0.5) * 60; $diff = $diff == 0 ? 60 : ($diff > 3600 ? 3600 : $diff); $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME, time() + $diff); return false; } else { $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isLocked); } } } else { return false; } } //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) { try { $p = PlatformFactory::NewPlatform($dbServer->platform); $environment = $dbServer->GetEnvironmentObject(); if (!$environment->isPlatformEnabled($dbServer->platform)) { throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id)); } if ($dbServer->GetCloudServerID()) { $serverHistory = $dbServer->getServerHistory(); $isTermination = in_array($dbServer->status, [SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE]); $isSuspension = in_array($dbServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND]); /* @var $terminationData TerminationData */ $terminationData = null; //NOTE: in any case, after call, be sure to set callback to null $this->setupClientCallback($p, function ($request, $response) use($dbServer, &$terminationData) { $terminationData = new TerminationData(); $terminationData->serverId = $dbServer->serverId; if ($request instanceof \http\Client\Request) { $terminationData->requestUrl = $request->getRequestUrl(); $terminationData->requestQuery = $request->getQuery(); $terminationData->request = $request->toString(); } if ($response instanceof \http\Client\Response) { $terminationData->response = $response->toString(); $terminationData->responseCode = $response->getResponseCode(); $terminationData->responseStatus = $response->getResponseStatus(); } }, $dbServer); try { $status = $dbServer->GetRealStatus(); } catch (Exception $e) { //eliminate callback $this->setupClientCallback($p, null, $dbServer); throw $e; } //eliminate callback $this->setupClientCallback($p, null, $dbServer); if ($dbServer->isCloudstack()) { //Workaround for when expunge flag not working and servers stuck in Destroyed state. $isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed'; } else { $isTerminated = $status->isTerminated(); } if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) { try { if ($dbServer->farmId != 0) { try { if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]])); if ($dbServer->index == 1 && $serversCount > 1) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); return false; } } } catch (Exception $e) { } \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); } } catch (Exception $e) { $this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage()); } $terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME); if (!$terminationTime || time() - $terminationTime > 180) { if ($isTermination) { $p->TerminateServer($dbServer); } else { $p->SuspendServer($dbServer); } $dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time()); if ($dbServer->farmId) { $wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown')); if (!$wasHostDownFired) { $event = new HostDownEvent($dbServer); $event->isSuspended = !$isTermination; \Scalr::FireEvent($dbServer->farmId, $event); } } } } else { if ($dbServer->status == SERVER_STATUS::TERMINATED) { if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) { $errorResolution = true; $serverHistory->setTerminated(); $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } } else { if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) { $dbServer->updateStatus(SERVER_STATUS::TERMINATED); $errorResolution = true; } else { if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) { $dbServer->update(['status' => SERVER_STATUS::SUSPENDED, 'remoteIp' => '', 'localIp' => '']); $errorResolution = true; } } } if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) { //Automatic error resolution $ste->delete(); } } } else { //If there is no cloudserverID we don't need to add this server into server history. //$serverHistory->setTerminated(); $dbServer->Remove(); } } catch (InstanceNotFound $e) { if ($serverHistory) { $serverHistory->setTerminated(); } $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } catch (Exception $e) { if ($request->serverId && stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false && $dbServer && $dbServer->platform == \SERVER_PLATFORMS::EC2) { // server has disableApiTermination flag on cloud, update server properties $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, 1); } else { if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || CloudPlatformSuspensionInfo::isSuspensionException($e) || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) { //Postpones unsuccessful task for 30 minutes. $ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage()); $minutes = rand(30, 40); $ste->retryAfter = new \DateTime('+' . $minutes . ' minutes'); if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { //We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } $ste->save(); } else { $this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage()); throw $e; } } } } return $request; }
public function StartThread($bundle_task_info) { $db = Core::GetDBInstance(); // Reconfigure observers; Scalr::ReconfigureObservers(); $BundleTask = BundleTask::LoadById($bundle_task_info['id']); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); } catch (ServerNotFoundException $e) { if (!$BundleTask->snapshotId) { $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED; $BundleTask->setDate('finished'); $BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId); $BundleTask->Save(); return; } } catch (Exception $e) { //$this->Logger->error($e->getMessage()); } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } // IF server is in pensing state $status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer); $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); if ($status->isPending()) { $BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName())); exit; } elseif ($status->isTerminated()) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } break; } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $DBServer->remoteIp = $ips['remoteIp']; $DBServer->localIp = $ips['locateIp']; $DBServer->save(); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV; $BundleTask->save(); $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: $BundleTask->Log(sprintf(_("Initializing SSH2 session to the server"))); try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, 22); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage())); //TODO: Set status of bundle log to failed exit; } //Prepare script $BundleTask->Log(sprintf(_("Uploading builder scripts..."))); $behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR); try { $options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $DBServer->platform, 'behaviour' => trim(str_replace("base", "", $behaviors)), 'queryenv-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/query-env", 'messaging-p2p.producer-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/messaging"); $command = 'scalarizr --import -y'; foreach ($options as $k => $v) { $command .= sprintf(' -o %s=%s', $k, $v); } if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') { $recipes = 'mysql=percona'; } else { $recipes = ''; } $scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH); $scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl"); $scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_NODE_NAME%", "\r\n"), array($DBServer->platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', '', '', '', '', '', '', "\n"), $scriptContents); if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) { throw new Exception("Cannot upload script"); } $BundleTask->Log(sprintf(_("Uploading chef recipes..."))); if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) { throw new Exception("Cannot upload chef recipes"); } } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $BundleTask->Log("Launching role builder routines on server"); $ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh"); $ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE; $BundleTask->save(); break; case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, 22); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $log = $ssh2Client->getFile('/var/log/role-builder-output.log'); $log_lines = explode("\r\n", $log); $last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE); while ($msg = trim(array_shift($log_lines))) { if (substr($msg, -1, 1) != ']') { continue; } if ($last_msg) { if ($msg != $last_msg) { continue; } elseif ($msg == $last_msg) { $last_msg = null; continue; } } if (stristr($msg, '[ Failed ]')) { $stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log'); $BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog)); $BundleTask->SnapshotCreationFailed($msg); /** Terminate server **/ PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $DBServer->status = SERVER_STATUS::PENDING_TERMINATE; $DBServer->save(); $BundleTask->Log(sprintf("Temporary server '%s' (%s) has been terminated", $DBServer->serverId, $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))); } else { $BundleTask->Log($msg); $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg); } } //Read /var/log/role-builder-output.log break; case SERVER_SNAPSHOT_CREATION_STATUS::PENDING: try { $platformModule = PlatformFactory::NewPlatform($BundleTask->platform); $platformModule->CreateServerSnapshot($BundleTask); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage()); $BundleTask->SnapshotCreationFailed($e->getMessage()); } break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING: $addedTime = strtotime($BundleTask->dateAdded); if ($addedTime + 3600 < time()) { $BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time."); } break; case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS: PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask); break; case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS: $r_farm_roles = array(); $BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType)); try { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); } catch (Exception $e) { if (stristr($e->getMessage(), "not found in database")) { $BundleTask->SnapshotCreationFailed("Farm was removed before task was finished"); } return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { try { $r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); } catch (Exception $e) { } } elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND new_role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { try { $r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']); } catch (Exception $e) { } } } $update_farm_dns_zones = array(); $completed_roles = 0; foreach ($r_farm_roles as $DBFarmRole) { if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) { $BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); $completed_roles++; } else { $servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)); $BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); if (count($servers) == 0) { $DBFarmRole->RoleID = $DBFarmRole->NewRoleID; $DBFarmRole->NewRoleID = null; $DBFarmRole->Save(); $update_farm_dns_zones[$DBFarmRole->FarmID] = 1; $completed_roles++; } else { $metaData = $BundleTask->getSnapshotDetails(); foreach ($servers as $server) { try { $DBServer = DBServer::LoadByID($server['server_id']); } catch (Exception $e) { //TODO: continue; } if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) { $DBServer->roleId = $BundleTask->roleId; $DBServer->Save(); if ($metaData['noServersReplace']) { $BundleTask->Log(sprintf("'Do not replace servers' option was checked. Server '%s' won't be replaced to new image.", $DBServer->serverId)); } else { $BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, already has all modifications. No need to replace it.", $DBServer->serverId)); } if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) { PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t\t\t\t\t\t", array(sprintf("Farm was in 'Synchronizing' state. Server terminated when bundling was completed. Bundle task #%s", $BundleTask->id), $DBServer->serverId)); } } else { if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?)", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE))) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID); $nDBServer = Scalr::LaunchServer($ServerCreateInfo); $nDBServer->replaceServerID = $DBServer->serverId; $nDBServer->Save(); $BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId)); } } // if serverid != bundletask->serverID } // foreach server } // count($servers) } } if ($completed_roles == count($r_farm_roles)) { $BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Save(); } try { if (count($update_farm_dns_zones) != 0) { foreach ($update_farm_dns_zones as $farm_id => $v) { $dnsZones = DBDNSZone::loadByFarmId($farm_id); foreach ($dnsZones as $dnsZone) { if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) { $dnsZone->updateSystemRecords(); $dnsZone->save(); } } } } } catch (Exception $e) { $this->Logger->fatal("DNS ZONE: {$e->getMessage()}"); } break; case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE: try { if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $saveOldRole = false; try { $dbRole = DBRole::loadById($DBServer->roleId); if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) { $saveOldRole = true; } } catch (Exception $e) { //NO OLD ROLE } if ($dbRole && $saveOldRole) { if ($DBServer) { $new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer); } else { $new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999); } $dbRole->name = $new_role_name; $BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name)); $dbRole->save(); } else { //TODO: //$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})"); } } try { $DBRole = DBRole::createFromBundleTask($BundleTask); } catch (Exception $e) { $BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again."); return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) { $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); if ($DBServer->status == SERVER_STATUS::IMPORTING) { if ($DBServer->farmId) { // Create DBFarmRole object // TODO: create DBFarm role } //$DBServer->Delete(); } } catch (Exception $e) { } } else { try { $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS)); if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); $DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } else { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { $DBFarmRole = DBFarmRole::LoadByID($farm_role['id']); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS; } catch (Exception $e) { $this->Logger->error($e->getMessage()); $BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage())); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; } } $BundleTask->Save(); } catch (Exception $e) { $this->Logger->error($e->getMessage()); } break; } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); try { $dbFarm = DBFarm::LoadByID($request->farmId); $curDate = new DateTime(); $tdValue = $dbFarm->GetSetting(Entity\FarmSetting::LEASE_TERMINATE_DATE); if ($tdValue) { $td = new DateTime($tdValue); if ($td < $curDate) { //Terminates farm SettingEntity::increase(SettingEntity::LEASE_TERMINATE_FARM); //Ajdusts both account & environment for the audit log \Scalr::getContainer()->auditlogger->setAccountId($dbFarm->ClientID)->setAccountId($dbFarm->EnvID); \Scalr::FireEvent($request->farmId, new FarmTerminatedEvent(0, 1, false, 1, true, null)); $this->log('INFO', sprintf('Farm: %s [ID: %d] was terminated by lease manager', $dbFarm->Name, $dbFarm->ID)); } else { // only inform user $days = $td->diff($curDate)->days; $notifications = json_decode($dbFarm->GetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND), true); $governance = new Scalr_Governance($dbFarm->EnvID); $settings = $governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'notifications'); if (is_array($settings)) { foreach ($settings as $n) { if (!$notifications[$n['key']] && $n['period'] >= $days) { $mailer = \Scalr::getContainer()->mailer; $tdHuman = Scalr_Util_DateTime::convertDateTime($td, $dbFarm->GetSetting(Entity\FarmSetting::TIMEZONE), 'M j, Y'); /* @var $user Entity\Account\User */ if ($n['to'] == 'owner') { if ($dbFarm->ownerId) { $user = Entity\Account\User::findPk($dbFarm->ownerId); } else { $user = Entity\Account\User::findOne([['accountId' => $dbFarm->ClientID], ['type' => Entity\Account\User::TYPE_ACCOUNT_OWNER]]); } if (\Scalr::config('scalr.auth_mode') == 'ldap') { $email = $user->getSetting(Entity\Account\User\UserSetting::NAME_LDAP_EMAIL); if (!$email) { $email = $user->email; } } else { $email = $user->email; } $mailer->addTo($email); } else { foreach (explode(',', $n['emails']) as $email) { $mailer->addTo(trim($email)); } } $mailer->sendTemplate(SCALR_TEMPLATES_PATH . '/emails/farm_lease_terminate.eml.php', array('terminateDate' => $tdHuman, 'farm' => $dbFarm->Name, 'envName' => $dbFarm->GetEnvironmentObject()->name, 'envId' => $dbFarm->GetEnvironmentObject()->id, 'showOwnerWarning' => !$dbFarm->ownerId)); $notifications[$n['key']] = 1; $dbFarm->SetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND, json_encode($notifications)); $this->log('INFO', "Notification was sent by key: %s about farm: %s [ID: %d] by lease manager", $n['key'], $dbFarm->Name, $dbFarm->ID); } } } } } } catch (Exception $e) { throw $e; } return $request; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $serverId = $request->serverId; $logger = Logger::getLogger(__CLASS__); $this->log("INFO", "Processing messages for %s server", $serverId); try { $dbserver = DBServer::LoadByID($serverId); if ($dbserver->farmId) { if ($dbserver->GetFarmObject()->Status == FARM_STATUS::TERMINATED) { throw new ServerNotFoundException("Farm related to this server has been terminated."); } } } catch (ServerNotFoundException $e) { //By some reason server does not exist $this->db->Execute("\n DELETE m FROM messages m\n WHERE m.server_id = ? AND m.`type` = ? AND m.`status` = ?\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]); return false; } //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); $rs = $this->db->Execute("\n SELECT m.* FROM messages m\n WHERE m.server_id = ? AND m.type = ? AND m.status = ?\n ORDER BY m.dtadded ASC\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]); while ($row = $rs->FetchRow()) { try { if ($row["message_format"] == 'xml') { $message = $this->serializer->unserialize($row["message"]); } else { $message = $this->jsonSerializer->unserialize($row["message"]); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_MESSAGE_FORMAT, 'json'); } $message->messageIpAddress = $row['ipaddress']; $event = null; $startTime = microtime(true); // Update scalarizr package version if ($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]) { $dbserver->setScalarizrVersion($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]); } if ($message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]) { $dbserver->SetProperty(SERVER_PROPERTIES::SZR_UPD_CLIENT_VERSION, $message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]); } if ($dbserver->GetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES)) { continue; } if ($message instanceof \Scalr_Messaging_Msg) { $this->log('INFO', "Handling '%s' for '%s' server", $message->getName(), $serverId); } try { if ($message instanceof Scalr_Messaging_Msg_OperationResult) { if ($message->status == 'ok' || $message->status == 'completed') { if ($message->name == 'Grow MySQL/Percona data volume' || $message->name == 'mysql.grow-volume') { $volumeConfig = $message->data ? $message->data : $message->result; $oldVolumeId = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::VOLUME_ID); $engine = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::DATA_STORAGE_ENGINE); try { $storageVolume = Scalr_Storage_Volume::init(); try { $storageVolume->loadById($volumeConfig->id); $storageVolume->setConfig($volumeConfig); $storageVolume->save(); } catch (Exception $e) { if (strpos($e->getMessage(), 'not found')) { $storageVolume->loadBy(array('id' => $volumeConfig->id, 'client_id' => $dbserver->clientId, 'env_id' => $dbserver->envId, 'name' => "'{$volumeConfig->tags->service}' data volume", 'type' => $engine, 'platform' => $dbserver->platform, 'size' => $volumeConfig->size, 'fstype' => $volumeConfig->fstype, 'purpose' => $volumeConfig->tags->service, 'farm_roleid' => $dbserver->farmRoleId, 'server_index' => $dbserver->index)); $storageVolume->setConfig($volumeConfig); $storageVolume->save(true); } else { throw $e; } } $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::VOLUME_ID, $volumeConfig->id, DBFarmRole::TYPE_LCL); if ($engine == MYSQL_STORAGE_ENGINE::EBS) { $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_EBS_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG); } elseif ($engine == MYSQL_STORAGE_ENGINE::RAID_EBS) { $dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_RAID_DISK_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG); } // Remove old $storageVolume->delete($oldVolumeId); } catch (Exception $e) { Logger::getLogger(__CLASS__)->error(new FarmLogMessage($dbserver->farmId, "Cannot save storage volume: {$e->getMessage()}")); } } } elseif ($message->status == 'error' || $message->status == 'failed') { if ($message->name == 'Initialization' || $message->name == 'system.init') { $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED, 1); if (is_object($message->error)) { $errorText = $message->error->message; } elseif ($message->error) { $errorText = $message->error; } $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText); $event = new HostInitFailedEvent($dbserver, $errorText); } } } elseif ($message instanceof Scalr_Messaging_Msg_InitFailed) { $errorText = $message->reason; $dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText); $event = new HostInitFailedEvent($dbserver, $errorText); } elseif ($message instanceof \Scalr_Messaging_Msg_RuntimeError) { $logger->fatal(new FarmLogMessage($dbserver->farmId, "Scalarizr failed to launch on server '{$dbserver->getNameByConvention()}' with runtime error: {$message->message}", $dbserver->serverId)); } elseif ($message instanceof Scalr_Messaging_Msg_UpdateControlPorts) { $apiPort = $message->api; $ctrlPort = $message->messaging; $snmpPort = $message->snmp; // Check API port; $currentApiPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_API_PORT); if (!$currentApiPort) { $currentApiPort = 8010; } if ($apiPort && $apiPort != $currentApiPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr API port was changed from {$currentApiPort} to {$apiPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_API_PORT, $apiPort); } // Check Control port $currentCtrlPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT); if (!$currentCtrlPort) { $currentCtrlPort = 8013; } if ($ctrlPort && $ctrlPort != $currentCtrlPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr Control port was changed from {$currentCtrlPort} to {$ctrlPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT, $ctrlPort); } //Check SNMP port $currentSnmpPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT); if (!$currentSnmpPort) { $currentSnmpPort = 8014; } if ($snmpPort && $snmpPort != $currentSnmpPort) { $logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr SNMP port was changed from {$currentSnmpPort} to {$snmpPort}", $dbserver->serverId)); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, $snmpPort); } } elseif ($message instanceof Scalr_Messaging_Msg_Win_HostDown) { $status = PlatformFactory::NewPlatform($dbserver->platform)->GetServerRealStatus($dbserver); if ($status->isRunning()) { $event = new RebootBeginEvent($dbserver); } else { if ($dbserver->platform == SERVER_PLATFORMS::EC2) { if (!$status->isTerminated()) { //Stopping $logger->error(new FarmLogMessage($dbserver->farmId, "Server is in '{$status->getName()}' state. Ignoring HostDown event.", $dbserver->serverId)); $isStopping = true; } } if ($isStopping) { $dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $dbserver->remoteIp = ""; $dbserver->localIp = ""; $dbserver->status = SERVER_STATUS::SUSPENDED; $dbserver->Save(); } $event = new HostDownEvent($dbserver); $event->isSuspended = true; } } elseif ($message instanceof Scalr_Messaging_Msg_Win_PrepareBundleResult) { try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); } catch (Exception $e) { } if ($bundleTask) { if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::PREPARING) { if ($message->status == 'ok') { $metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'os' => $message->os, 'software' => $message->software); $bundleTask->setMetaData($metaData); $bundleTask->Save(); PlatformFactory::NewPlatform($bundleTask->platform)->CreateServerSnapshot($bundleTask); } else { $bundleTask->SnapshotCreationFailed("PrepareBundle procedure failed: {$message->lastError}"); } } } } elseif ($message instanceof Scalr_Messaging_Msg_DeployResult) { try { $deploymentTask = Scalr_Model::init(Scalr_Model::DM_DEPLOYMENT_TASK)->loadById($message->deployTaskId); } catch (Exception $e) { } if ($deploymentTask) { if ($message->status == 'error') { $deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_FAILED; $deploymentTask->lastError = $message->lastError; } else { $deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_DEPLOYED; $deploymentTask->dtDeployed = date("Y-m-d H:i:s"); } $deploymentTask->save(); } } elseif ($message instanceof Scalr_Messaging_Msg_Hello) { $event = $this->onHello($message, $dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_FireEvent) { //Validate event $isEventExist = $this->db->GetOne("\n SELECT id FROM event_definitions\n WHERE name = ? AND ((env_id = ? AND account_id = ?) OR (env_id IS NULL AND account_id = ?) OR (env_id IS NULL AND account_id IS NULL))\n LIMIT 1\n ", array($message->eventName, $dbserver->envId, $dbserver->clientId, $dbserver->clientId)); if ($isEventExist) { $event = new CustomEvent($dbserver, $message->eventName, (array) $message->params); } } elseif ($message instanceof Scalr_Messaging_Msg_HostUpdate) { try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_MongoDb) { /********* MONGODB *********/ try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_DbMsr) { /********* DBMSR *********/ try { $dbFarmRole = $dbserver->GetFarmRoleObject(); } catch (Exception $e) { } if ($dbFarmRole instanceof DBFarmRole) { foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) { $behavior->handleMessage($message, $dbserver); } } } elseif ($message instanceof Scalr_Messaging_Msg_HostInit) { $event = $this->onHostInit($message, $dbserver); try { $dbserver->updateTimelog('ts_hi', $message->secondsSinceBoot, $message->secondsSinceStart); } catch (Exception $e) { } if (!$event) { continue; } } elseif ($message instanceof Scalr_Messaging_Msg_HostUp) { $event = $this->onHostUp($message, $dbserver); try { $dbserver->updateTimelog('ts_hu'); } catch (Exception $e) { } } elseif ($message instanceof Scalr_Messaging_Msg_HostDown) { $ignoreHostDown = false; $p = PlatformFactory::NewPlatform($dbserver->platform); $status = $p->GetServerRealStatus($dbserver); if ($dbserver->isOpenstack()) { if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) { $logger->error(new FarmLogMessage($dbserver->farmId, "Rackspace server is in " . $status->getName() . " state. Ignoring HostDown message.", $dbserver->serverId)); $isRebooting = true; } } elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) { if ($status->getName() == 'STOPPING') { // We don't know is this shutdown or stop so let's ignore HostDown // and wait for status change $doNotProcessMessage = true; $ignoreHostDown = true; } elseif ($status->getName() == 'RUNNING') { $isRebooting = true; } elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) { $isStopping = true; } } else { if ($p->getResumeStrategy() == 'init') { //TODO: Check is is stopping or shutting-down procedure. if (!$status->isTerminated()) { $isStopping = true; } } } if ($isStopping) { $dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $dbserver->remoteIp = ""; $dbserver->localIp = ""; $dbserver->status = SERVER_STATUS::SUSPENDED; $dbserver->Save(); $event = new HostDownEvent($dbserver); $event->isSuspended = true; } elseif ($isRebooting) { $event = new RebootBeginEvent($dbserver); } elseif (!$ignoreHostDown) { if ($dbserver->farmId) { $wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown')); if (!$wasHostDownFired) { $event = new HostDownEvent($dbserver); } } } } elseif ($message instanceof Scalr_Messaging_Msg_RebootStart) { $event = new RebootBeginEvent($dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_RebootFinish) { if (!$dbserver->localIp && $message->localIp) { $dbserver->localIp = $message->localIp; $dbserver->Save(); } $event = new RebootCompleteEvent($dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_BeforeHostUp) { $event = new BeforeHostUpEvent($dbserver); try { $dbserver->updateTimelog('ts_bhu'); } catch (Exception $e) { } } elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceAttached) { if ($dbserver->platform == SERVER_PLATFORMS::EC2) { $aws = $dbserver->GetEnvironmentObject()->aws($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION)); $instanceId = $dbserver->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID); //The main goal of using filters there is to considerably decrease the size of the response. $volumes = $aws->ec2->volume->describe(null, array(array('name' => VolumeFilterNameType::attachmentInstanceId(), 'value' => (string) $instanceId), array('name' => VolumeFilterNameType::attachmentDevice(), 'value' => (string) $message->deviceName), array('name' => VolumeFilterNameType::status(), 'value' => AMAZON_EBS_STATE::IN_USE))); foreach ($volumes as $volume) { /* @var $volume Scalr\Service\Aws\Ec2\DataType\VolumeData */ if ($volume->status == AMAZON_EBS_STATE::IN_USE && count($volume->attachmentSet) && $volume->attachmentSet[0]->instanceId == $instanceId && $volume->attachmentSet[0]->device == $message->deviceName) { $message->volumeId = $volume->volumeId; } } //Releases memory unset($volumes); $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } $event = new EBSVolumeAttachedEvent($dbserver, $message->deviceName, $message->volumeId); } elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceMounted) { // Single volume $ebsinfo = $this->db->GetRow("\n SELECT * FROM ec2_ebs WHERE volume_id=? LIMIT 1\n ", array($message->volumeId)); if ($ebsinfo) { $this->db->Execute("\n UPDATE ec2_ebs\n SET mount_status=?, isfsexist='1'\n WHERE id=?\n ", array(EC2_EBS_MOUNT_STATUS::MOUNTED, $ebsinfo['id'])); } $event = new EBSVolumeMountedEvent($dbserver, $message->mountpoint, $message->volumeId, $message->deviceName); } elseif ($message instanceof Scalr_Messaging_Msg_RebundleResult) { if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_OK) { $metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'dist' => $message->dist, 'os' => $message->os, 'software' => $message->software); if ($dbserver->platform == SERVER_PLATFORMS::EC2) { if ($message->aws) { if ($message->aws->rootDeviceType == 'ebs') { $tags[] = ROLE_TAGS::EC2_EBS; } if ($message->aws->virtualizationType == 'hvm') { $tags[] = ROLE_TAGS::EC2_HVM; } } else { $aws = $dbserver->GetEnvironmentObject()->aws($dbserver); try { $info = $aws->ec2->image->describe($dbserver->GetProperty(EC2_SERVER_PROPERTIES::AMIID))->get(0); if ($info->rootDeviceType == 'ebs') { $tags[] = ROLE_TAGS::EC2_EBS; } else { try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) { $tags[] = ROLE_TAGS::EC2_EBS; } } catch (Exception $e) { } } if ($info->virtualizationType == 'hvm') { $tags[] = ROLE_TAGS::EC2_HVM; } unset($info); } catch (Exception $e) { $metaData['tagsError'] = $e->getMessage(); try { $bundleTask = BundleTask::LoadById($message->bundleTaskId); if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) { $tags[] = ROLE_TAGS::EC2_EBS; } } catch (Exception $e) { } } //Releases memory $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } } $metaData['tags'] = $tags; $event = new RebundleCompleteEvent($dbserver, $message->snapshotId, $message->bundleTaskId, $metaData); } else { if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_FAILED) { $event = new RebundleFailedEvent($dbserver, $message->bundleTaskId, $message->lastError); } } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateDataBundleResult) { if ($message->status == "ok") { $event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE, array('snapshotConfig' => $message->snapshotConfig, 'logFile' => $message->logFile, 'logPos' => $message->logPos, 'dataBundleSize' => $message->dataBundleSize, 'snapshotId' => $message->snapshotId)); } else { $event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE); $event->lastError = $message->lastError; } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateBackupResult) { if ($message->status == "ok") { $event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP, array()); $event->backupParts = $message->backupParts; } else { $event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP); $event->lastError = $message->lastError; } } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_PromoteToMasterResult) { $event = $this->onMysql_PromoteToMasterResult($message, $dbserver); } elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreatePmaUserResult) { $farmRole = DBFarmRole::LoadByID($message->farmRoleId); if ($message->status == "ok") { $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_USER, $message->pmaUser, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_PASS, $message->pmaPassword, DBFarmRole::TYPE_LCL); } else { $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_ERROR, $message->lastError, DBFarmRole::TYPE_LCL); } } elseif ($message instanceof Scalr_Messaging_Msg_RabbitMq_SetupControlPanelResult) { $farmRole = $dbserver->GetFarmRoleObject(); if ($message->status == "ok") { $mgmtHost = $dbserver->getSzrHost(); if ($message->port) { $mgmtURL = "http://{$mgmtHost}:{$message->port}/mgmt"; } elseif ($message->cpanelUrl) { $info = parse_url($message->cpanelUrl); $mgmtURL = "http://{$mgmtHost}:{$info['port']}/mgmt"; } $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, $dbserver->serverId, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_URL, $mgmtURL, DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); } else { $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL); $farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_ERROR_MSG, $message->lastError, DBFarmRole::TYPE_LCL); } } elseif ($message instanceof Scalr_Messaging_Msg_AmiScriptsMigrationResult) { try { //Open security group: if ($dbserver->platform == SERVER_PLATFORMS::EC2) { $info = PlatformFactory::NewPlatform($dbserver->platform)->GetServerExtendedInformation($dbserver); $sg = explode(", ", $info['Security groups']); foreach ($sg as $sgroup) { if ($sgroup != 'default') { // For Scalarizr $group_rules = array(array('rule' => 'tcp:8013:8013:0.0.0.0/0'), array('rule' => 'udp:8014:8014:0.0.0.0/0')); $aws = $dbserver->GetEnvironmentObject()->aws($dbserver); $ipPermissions = new \Scalr\Service\Aws\Ec2\DataType\IpPermissionList(); foreach ($group_rules as $rule) { $group_rule = explode(":", $rule["rule"]); $ipPermissions->append(new \Scalr\Service\Aws\Ec2\DataType\IpPermissionData($group_rule[0], $group_rule[1], $group_rule[2], new \Scalr\Service\Aws\Ec2\DataType\IpRangeData($group_rule[3]))); } $aws->ec2->securityGroup->authorizeIngress($ipPermissions, null, $sgroup); $dbserver->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); unset($ipPermissions); break; } } } } catch (Exception $e) { $logger->fatal($e->getMessage()); } $dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, 8014); $dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, "0.7.217"); if ($message->mysql) { $event = $this->onHostUp($message, $dbserver, true); } } $handle_status = MESSAGE_STATUS::HANDLED; } catch (Exception $e) { $handle_status = MESSAGE_STATUS::FAILED; $logger->error(sprintf("Cannot handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s). %s", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId, $e->getMessage() . "({$e->getFile()}:{$e->getLine()})")); } if (!$doNotProcessMessage) { $totalTime = microtime(true) - $startTime; $this->db->Execute("\n UPDATE messages\n SET status = ?, processing_time = ?, dtlasthandleattempt = NOW()\n WHERE messageid = ?\n ", array($handle_status, $totalTime, $message->messageId)); } else { $logger->info(sprintf("Handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s) is postponed due to status transition", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId)); } if ($event) { \Scalr::FireEvent($dbserver->farmId, $event); } } catch (Exception $e) { $logger->error($e->getMessage()); } } return $request; }
public function StartThread($bundle_task_info) { $db = \Scalr::getDb(); // Reconfigure observers; Scalr::ReconfigureObservers(); $BundleTask = BundleTask::LoadById($bundle_task_info['id']); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); } catch (\Scalr\Exception\ServerNotFoundException $e) { if (!$BundleTask->snapshotId) { $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED; $BundleTask->setDate('finished'); $BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId); $BundleTask->Save(); return; } } catch (Exception $e) { //$this->Logger->error($e->getMessage()); } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION: $ctrlPort = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT); if (!$ctrlPort) { $ctrlPort = 8013; } if (\Scalr::config('scalr.instances_connection_policy') == 'local') { $requestHost = $DBServer->localIp; } elseif (\Scalr::config('scalr.instances_connection_policy') == 'public') { $requestHost = $DBServer->remoteIp; } elseif (\Scalr::config('scalr.instances_connection_policy') == 'auto') { if ($DBServer->remoteIp) { $requestHost = $DBServer->remoteIp; } else { $requestHost = $DBServer->localIp; } } $conn = @fsockopen($requestHost, $ctrlPort, $errno, $errstr, 10); if ($conn) { $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1); $BundleTask->Log("Outbound connection successfully established. Awaiting user action: prebuild automation selection"); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION; $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); $BundleTask->Save(); } else { $errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $requestHost, $ctrlPort, $errstr); $errMsg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR); if (!$errMsg || $errstr != $errMsg) { $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr); $BundleTask->Log("{$errstr} Will try again in a few minutes."); } } exit; break; case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION: //NOTHING TO DO; exit; break; case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $BundleTask->setDate('started'); case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: if (!PlatformFactory::NewPlatform($DBServer->platform)->GetServerID($DBServer)) { $BundleTask->Log(sprintf(_("Waiting for temporary server"))); exit; } if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) { $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } // IF server is in pensing state $status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer); if ($status->isPending()) { $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); $BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName())); exit; } elseif ($status->isTerminated()) { $BundleTask->Log(sprintf(_("Server status: %s"), $status->getName())); $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->save(); $BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); exit; } break; } switch ($BundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $DBServer->remoteIp = $ips['remoteIp']; $DBServer->localIp = $ips['localIp']; $DBServer->save(); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV; $BundleTask->save(); $BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status)); break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: $BundleTask->Log(sprintf(_("Initializing SSH2 session to the server"))); if ($DBServer->platform == SERVER_PLATFORMS::IDCF && !$DBServer->remoteIp) { try { $BundleTask->Log("Creating port forwarding rules to be able to connect to the server by SSH"); $environment = $DBServer->GetEnvironmentObject(); $cloudLocation = $DBServer->GetCloudLocation(); $platform = PlatformFactory::NewPlatform($DBServer->platform); $sharedIpId = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SHARED_IP_ID . ".{$cloudLocation}", $environment, false); $sharedIp = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SHARED_IP . ".{$cloudLocation}", $environment, false); $BundleTask->Log("Shared IP: {$sharedIp}"); $cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $environment), $DBServer->platform); // Create port forwarding rules for scalarizr $port = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false); if (!$port) { $port1 = 30000; $port2 = 30001; $port3 = 30002; $port4 = 30003; } else { $port1 = $port + 1; $port2 = $port1 + 1; $port3 = $port2 + 1; $port4 = $port3 + 1; } $result2 = $cs->createPortForwardingRule($sharedIpId, 8014, "udp", $port1, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)); $result1 = $cs->createPortForwardingRule($sharedIpId, 8013, "tcp", $port1, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)); $result3 = $cs->createPortForwardingRule($sharedIpId, 8010, "tcp", $port3, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)); $result4 = $cs->createPortForwardingRule($sharedIpId, 8008, "tcp", $port2, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)); $result5 = $cs->createPortForwardingRule($sharedIpId, 22, "tcp", $port4, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID)); $DBServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4)); $DBServer->remoteIp = $sharedIp; $DBServer->Save(); $platform->setConfigVariable(array(Modules_Platforms_Cloudstack::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false); } catch (Exception $e) { $BundleTask->Log("Unable to create port-forwarding rules: {$e->getMessage()}"); } exit; } if ($DBServer->platform == SERVER_PLATFORMS::ECS && !$DBServer->remoteIp) { $BundleTask->Log(sprintf(_("Server doesn't have public IP. Assigning..."))); $osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION)); $ports = $osClient->network->ports->list(); foreach ($ports as $port) { if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $serverNetworkPort = $port->id; break; } } $ips = $osClient->network->floatingIps->list(); //Check free existing IP $ipAssigned = false; $ipAddress = false; $ipId = false; $ipInfo = false; foreach ($ips as $ip) { if ($ip->port_id && $ip->port_id == $serverNetworkPort) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipAssigned = true; $ipInfo = $ip; break; } if (!$ip->fixed_ip_address && !$ipAddress) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipInfo = $ip; } } if (!$ipAssigned) { if (!$serverNetworkPort) { $BundleTask->Log("Unable to identify network port of instance"); exit; } else { if (!$ipAddress) { $networks = $osClient->network->listNetworks(); foreach ($networks as $network) { if ($network->{"router:external"} == true) { $publicNetworkId = $network->id; } } if (!$publicNetworkId) { $BundleTask->Log("Unable to identify public network to allocate"); exit; } else { $ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort); $ipAddress = $ip->floating_ip_address; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); $BundleTask->Log("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}"); } } else { $BundleTask->Log("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")"); $osClient->network->floatingIps->update($ipId, $serverNetworkPort); } } } else { $BundleTask->Log("IP: {$ipAddress} already assigned"); } if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } exit; } try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } //Prepare script $BundleTask->Log(sprintf(_("Uploading builder scripts..."))); $behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR); try { if ($DBServer->isOpenstack()) { $platform = SERVER_PLATFORMS::OPENSTACK; } else { $platform = $DBServer->platform; } $baseUrl = \Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host'); $options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $DBServer->envId, 'region' => $DBServer->GetCloudLocation(), 'scalr-id' => SCALR_ID); $command = 'scalarizr --import -y'; foreach ($options as $k => $v) { $command .= sprintf(' -o %s=%s', $k, $v); } if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') { $recipes = 'mysql=percona'; } else { $recipes = ''; } $scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH); $scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl"); /* %CHEF_SERVER_URL% %CHEF_VALIDATOR_NAME% %CHEF_VALIDATOR_KEY% %CHEF_ENVIRONMENT% %CHEF_ROLE_NAME% */ $chefServerId = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID); if ($chefServerId) { $chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId)); $chefServerInfo['v_auth_key'] = $this->crypto->decrypt($chefServerInfo['v_auth_key'], $this->cryptoKey); } $scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents); if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) { throw new Exception("Cannot upload script"); } /* $BundleTask->Log(sprintf(_("Uploading chef recipes..."))); if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) { throw new Exception("Cannot upload chef recipes"); } */ } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $BundleTask->Log("Launching role builder routines on server"); $ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh"); // For CGE we need to use sudo if ($BundleTask->platform == SERVER_PLATFORMS::GCE || $BundleTask->osFamily == 'amazon') { $shell = $ssh2Client->getShell(); fwrite($shell, "sudo su\n"); fwrite($shell, "touch /var/log/role-builder-output.log 2>&1\n"); fwrite($shell, "chmod 0666 /var/log/role-builder-output.log 2>&1\n"); fwrite($shell, "setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &\n"); // Get output $iterations = 0; while ($l = @fgets($shell, 4096) && $iterations < 10) { $meta = stream_get_meta_data($shell); if ($meta["timed_out"]) { break; } $retval .= $l; $iterations++; } @fclose($shell); $BundleTask->Log("Exec result: {$retval}"); /* $r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log"); $BundleTask->Log("1: {$r1} ({$ssh2Client->stdErr})"); $r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log"); $BundleTask->Log("2: {$r2} ({$ssh2Client->stdErr})"); $r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); $BundleTask->Log("3: {$r3} ({$ssh2Client->stdErr})"); */ } else { $ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE; $BundleTask->save(); break; case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: try { $ssh2Client = $DBServer->GetSsh2Client(); $ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed exit; } $log = $ssh2Client->getFile('/var/log/role-builder-output.log'); $log_lines = explode("\r\n", $log); $last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE); while ($msg = trim(array_shift($log_lines))) { if (substr($msg, -1, 1) != ']') { continue; } if ($last_msg) { if ($msg != $last_msg) { continue; } elseif ($msg == $last_msg) { $last_msg = null; continue; } } if (stristr($msg, '[ Failed ]')) { $stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log'); $BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog)); $BundleTask->SnapshotCreationFailed($msg); } else { $BundleTask->Log($msg); $DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg); } } //Read /var/log/role-builder-output.log break; case SERVER_SNAPSHOT_CREATION_STATUS::PENDING: try { $platformModule = PlatformFactory::NewPlatform($BundleTask->platform); $platformModule->CreateServerSnapshot($BundleTask); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage()); $BundleTask->SnapshotCreationFailed($e->getMessage()); } break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING: $addedTime = strtotime($BundleTask->dateAdded); if ($addedTime + 3600 < time()) { $BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time."); } break; case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS: PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask); break; case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS: $r_farm_roles = array(); $BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType)); try { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); } catch (Exception $e) { if (stristr($e->getMessage(), "not found in database")) { $BundleTask->SnapshotCreationFailed("Farm was removed before task was finished"); } return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { try { $r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); } catch (Exception $e) { } } elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=? AND new_role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { try { $r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']); } catch (Exception $e) { } } } $update_farm_dns_zones = array(); $completed_roles = 0; foreach ($r_farm_roles as $DBFarmRole) { if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) { $BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); $completed_roles++; } else { $servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)); $BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. " . "Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); if (count($servers) == 0) { $DBFarmRole->RoleID = $DBFarmRole->NewRoleID; $DBFarmRole->NewRoleID = null; $DBFarmRole->Save(); $update_farm_dns_zones[$DBFarmRole->FarmID] = 1; $completed_roles++; } else { $metaData = $BundleTask->getSnapshotDetails(); foreach ($servers as $server) { try { $DBServer = DBServer::LoadByID($server['server_id']); } catch (Exception $e) { //TODO: continue; } if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) { $DBServer->roleId = $BundleTask->roleId; $DBServer->Save(); if ($metaData['noServersReplace']) { $BundleTask->Log(sprintf("'Do not replace servers' option was checked. " . "Server '%s' won't be replaced to new image.", $DBServer->serverId)); } else { $BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, " . "already has all modifications. No need to replace it.", $DBServer->serverId)); } if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) { $DBServer->terminate(array('BUNDLE_TASK_FINISHED', 'Synchronizing', $BundleTask->id)); } } else { if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?,?) LIMIT 1", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TROUBLESHOOTING))) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID); $nDBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Server replacement after snapshotting", !empty($BundleTask->createdById) ? $BundleTask->createdById : null); $nDBServer->replaceServerID = $DBServer->serverId; $nDBServer->Save(); $BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId)); } } } } } } if ($completed_roles == count($r_farm_roles)) { $BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { if ($DBServer->status == SERVER_STATUS::IMPORTING) { $DBServer->Remove(); } elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) { $BundleTask->Log("Terminating temporary server"); $DBServer->terminate('TEMPORARY_SERVER_ROLE_BUILDER'); $BundleTask->Log("Termination request has been sent"); } } catch (Exception $e) { $BundleTask->Log("Warning: {$e->getMessage()}"); } $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Save(); } try { if (count($update_farm_dns_zones) != 0) { foreach ($update_farm_dns_zones as $farm_id => $v) { $dnsZones = DBDNSZone::loadByFarmId($farm_id); foreach ($dnsZones as $dnsZone) { if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) { $dnsZone->updateSystemRecords(); $dnsZone->save(); } } } } } catch (Exception $e) { $this->Logger->fatal("DNS ZONE: {$e->getMessage()}"); } break; case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE: try { if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $saveOldRole = false; try { $dbRole = DBRole::loadById($DBServer->roleId); if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) { $saveOldRole = true; } } catch (Exception $e) { //NO OLD ROLE } if ($dbRole && $saveOldRole) { if ($DBServer) { $new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer); } else { $new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999); } $dbRole->name = $new_role_name; $BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name)); $dbRole->save(); } else { //TODO: //$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})"); } } try { $DBRole = DBRole::createFromBundleTask($BundleTask); } catch (Exception $e) { $BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again."); return; } if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) { $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); try { $DBServer = DBServer::LoadByID($BundleTask->serverId); if ($DBServer->status == SERVER_STATUS::IMPORTING) { if ($DBServer->farmId) { // Create DBFarmRole object // TODO: create DBFarm role } } } catch (Exception $e) { } } else { try { $BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS)); if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { $DBFarm = DBFarm::LoadByID($BundleTask->farmId); $DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } else { $farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId)); foreach ($farm_roles as $farm_role) { $DBFarmRole = DBFarmRole::LoadByID($farm_role['id']); $DBFarmRole->NewRoleID = $BundleTask->roleId; $DBFarmRole->Save(); } } $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS; } catch (Exception $e) { $this->Logger->error($e->getMessage()); $BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage())); $BundleTask->setDate('finished'); $BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; } } if ($BundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) { try { if ($DBServer->status == SERVER_STATUS::IMPORTING) { $DBServer->Remove(); } elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) { $BundleTask->Log("Terminating temporary server"); $DBServer->terminate('TEMPORARY_SERVER_ROLE_BUILDER'); $BundleTask->Log("Termination request has been sent"); } } catch (Exception $e) { $BundleTask->Log("Warning: {$e->getMessage()}"); } } $BundleTask->Save(); } catch (Exception $e) { $this->Logger->error($e->getMessage()); } break; } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $db = \Scalr::getDb(); //Speed up poller if ($this->config()->daemon) { //Warming up static DI cache \Scalr::getContainer()->warmup(); } // Reconfigure observers \Scalr::ReconfigureObservers(); $DBFarm = DBFarm::LoadByID($request->farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $transactionId = abs(crc32(posix_getpid() . $request->farmId)); $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status); //Retrieves the number of either terminated or suspended servers for the farm $servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { //There are no servers for this farm return; } $this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID); $config = \Scalr::getContainer()->config; foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { /* @var $DBServer \DBServer */ try { if ($DBServer->cloudLocation) { try { $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId); $p = PlatformFactory::NewPlatform($DBServer->platform); $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } catch (Exception $e) { $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage()); continue; } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()); } if (!$serverInfo) { if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) { continue; } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId)); continue; } if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) { $action = 'terminate'; if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) { $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server"); } if ($action == 'flag') { $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1); } else { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); } continue; } } else { //http.persistent.handles.limit must be set to 0 for pecl-http version 1 $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId); //More debug $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache))); $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}")); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { /* @var $env \Scalr_Environment */ $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); //Saving the reason why this environment is disabled $env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]); return; } elseif (stristr($e->getMessage(), "Could not connect to host")) { continue; } $this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine()); continue; } try { if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { if ($openstackErrorState) { try { $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId)); } catch (Exception $e) { } } else { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); } $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { //In case the server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $DBServer->remoteIp = ""; $DBServer->localIp = ""; $DBServer->status = SERVER_STATUS::SUSPENDED; $DBServer->Save(); $event = new HostDownEvent($DBServer); $event->isSuspended = true; \Scalr::FireEvent($DBFarm->ID, $event); continue; } else { //If the server was suspended during initialization //we do not support this and need to terminate this instance $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { $platform = PlatformFactory::NewPlatform($DBServer->platform); if ($DBServer->platform == \SERVER_PLATFORMS::GCE) { if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') { continue; } } // For Openstack we need to re-accociate IPs try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]); } } if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) { if (!$DBServer->remoteIp) { $DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer); $DBServer->Save(); } } if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) { $DBServer->status = \SERVER_STATUS::PENDING; $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1); $DBServer->dateAdded = date("Y-m-d H:i:s"); $DBServer->Save(); } else { $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0); \Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, "")); } continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING) { if (!$DBServer->remoteIp && !$DBServer->localIp) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) { $DBServer->remoteIp = $ipaddresses['remoteIp']; $DBServer->localIp = $ipaddresses['localIp']; $DBServer->Save(); } //Add tags Ec2Helper::createServerTags($DBServer); } if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = Ec2EipHelper::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } } try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId)); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scripting_event, $DBServer->farmRoleId]); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId)); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false); } catch (Exception $err) { $this->getLogger()->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } //Whether IP address is changed if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId)); \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO Check health } } } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) { \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform); $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() < $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $this->getLogger()->fatal($e->getMessage()); } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(5); $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage()); } else { $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage()); } } } return $request; }
public function StartThread($mysql_farm_role) { // Reconfigure observers; Scalr::ReconfigureObservers(); $db = \Scalr::getDb(); $DBFarmRole = DBFarmRole::LoadByID($mysql_farm_role['id']); try { $DBFarm = $DBFarmRole->GetFarmObject(); } catch (Exception $e) { return; } //skip terminated farms if ($DBFarm->Status != FARM_STATUS::RUNNING) { return; } $tz = Scalr_Environment::init()->loadById($DBFarm->EnvID)->getPlatformConfigValue(Scalr_Environment::SETTING_TIMEZONE); if ($tz) { date_default_timezone_set($tz); } // // Check replication status // $this->Logger->info("[FarmID: {$DBFarm->ID}] Checking replication status"); $servers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::RUNNING)); // // Check backups and mysql bandle procedures // //Backups if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_ENABLED) && $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) != 0) { if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING) == 1) { // Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots) // We must wait for running bundle process. $bcp_timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) * 60 * 5; if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BCP_TS) + $bcp_timeout < time()) { $bcp_timeouted = true; } if (!empty($bcp_timeouted)) { $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING, 0, Entity\FarmRoleSetting::TYPE_LCL); $this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Backup already running. Timeout. Clear lock."); } } else { $timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) * 60; if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BCP_TS) + $timeout < time()) { $this->Logger->info("[FarmID: {$DBFarm->ID}] Need new backup"); $servers = $DBFarm->GetMySQLInstances(false, true); if (empty($servers[0])) { $servers = $DBFarm->GetMySQLInstances(true); } else { $servers = array_reverse($servers); } $DBServer = isset($servers[0]) ? $servers[0] : null; if ($DBServer) { if ($DBServer->status == SERVER_STATUS::RUNNING) { $msg = new Scalr_Messaging_Msg_Mysql_CreateBackup($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_ROOT_PASSWORD)); $DBServer->SendMessage($msg); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING, 1, Entity\FarmRoleSetting::TYPE_LCL); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_BCP_SERVER_ID, $DBServer->serverId, Entity\FarmRoleSetting::TYPE_LCL); } } else { $this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql instances for run backup procedure!"); } } } } if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_ENABLED) && $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY) != 0) { if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING) == 1) { // Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots) // We must wait for running bundle process. $bundle_timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY) * (3600 * 2); if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BUNDLE_TS) + $bundle_timeout < time()) { $bundle_timeouted = true; } if (!empty($bundle_timeouted)) { $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING, 0, Entity\FarmRoleSetting::TYPE_LCL); $this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Bundle already running. Timeout. Clear lock."); } } else { /* * Check bundle window */ $bundleEvery = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY); $timeout = $bundleEvery * 3600; $lastBundleTime = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BUNDLE_TS); $performBundle = false; if ($bundleEvery % 24 == 0) { if ($lastBundleTime) { $days = $bundleEvery / 24; $bundleDay = (int) date("md", strtotime("+{$days} day", $lastBundleTime)); if ($bundleDay > (int) date("md")) { return; } } $pbwFrom = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_MM)); $pbwTo = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_MM)); if ($pbwFrom && $pbwTo) { $current_time = (int) date("Hi"); if ($pbwFrom <= $current_time && $pbwTo >= $current_time) { $performBundle = true; } } else { $performBundle = true; } } else { //Check timeout if ($lastBundleTime + $timeout < time()) { $pbwFrom = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_MM)); $pbwTo = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_MM)); if ($pbwFrom && $pbwTo) { $current_time = (int) date("Hi"); if ($pbwFrom <= $current_time && $pbwTo >= $current_time) { $performBundle = true; } } else { $performBundle = true; } } } if ($performBundle) { $this->Logger->info("[FarmID: {$DBFarm->ID}] Need mySQL bundle procedure"); // Rebundle $servers = $DBFarm->GetMySQLInstances(true, false); $DBServer = isset($servers[0]) ? $servers[0] : null; if ($DBServer) { if ($DBServer->status == SERVER_STATUS::RUNNING) { $DBServer->SendMessage(new Scalr_Messaging_Msg_Mysql_CreateDataBundle()); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING, 1, Entity\FarmRoleSetting::TYPE_LCL); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_SERVER_ID, $DBServer->serverId, Entity\FarmRoleSetting::TYPE_LCL); } } else { $this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql master instances for run bundle procedure!"); } } } } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $db = \Scalr::getDb(); //The list of the suspension information about cloud platforms $this->aSuspensionInfo = []; //Speed up poller if ($this->config()->daemon) { //Warming up static DI cache \Scalr::getContainer()->warmup(); } // Reconfigure observers \Scalr::ReconfigureObservers(); $DBFarm = DBFarm::LoadByID($request->farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $transactionId = abs(crc32(posix_getpid() . $request->farmId)); $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform); $jobStartTime = microtime(true); //Retrieves the number of either terminated or suspended servers for the farm $servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers\n WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { //There are no servers for this farm and platform return; } $this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform); $config = \Scalr::getContainer()->config; /* if ($request->platform) { $p = PlatformFactory::NewPlatform($request->platform); $p->ClearCache(); } */ $p = PlatformFactory::NewPlatform($request->platform); foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) { /* @var $DBServer \DBServer */ //Get platform suspension info $suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId); //If the cloud platform is suspended we should not process it if ($suspensionInfo->isSuspended()) { continue; } try { //1. We need to check that server is exists in cloud and not missed. // (On Openstack server can be missed and should not be terminated) $cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation); if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) { try { $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform); if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) { //For Azure we need to pass resource group instead of cloudLocation $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP)); } else { $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } //We successfully polled cloud so can resume suspension status for the cloud platform if ($suspensionInfo->isPendingSuspend()) { $suspensionInfo->resume(); } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage()); continue; } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()); continue; } if (!$serverInfo) { if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) { continue; } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId)); continue; } $action = 'terminate'; if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) { $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server"); } if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]); } else { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); } continue; } } else { //http.persistent.handles.limit must be set to 0 for pecl-http version 1 $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]); } } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } $this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine()); continue; } try { if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { // If openstack server is in ERROR state we need more details if ($openstackErrorState) { try { $info = $p->GetServerExtendedInformation($DBServer); $status = empty($info['Status']) ? false : $info['Status']; } catch (Exception $e) { } } if (empty($status)) { $status = $DBServer->GetRealStatus()->getName(); } \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId)); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { //In case the server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); $event = new HostDownEvent($DBServer); $event->isSuspended = true; \Scalr::FireEvent($DBFarm->ID, $event); continue; } else { if ($DBServer->status != \SERVER_STATUS::RESUMING) { //If the server was suspended during initialization //we do not support this and need to terminate this instance if ($DBServer->platform == \SERVER_PLATFORMS::EC2) { try { $info = $p->GetServerExtendedInformation($DBServer); $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : ''; } catch (\Exception $e) { // no need to do anything here; } $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus); } $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } else { // Need to clear cache, because this situation happens only when cache is stale. $p->ClearCache(); } } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { if ($DBServer->platform == \SERVER_PLATFORMS::GCE) { if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') { continue; } } $update = []; // For Openstack we need to re-accociate IPs try { if ($DBServer->isOpenstack()) { OpenstackHelper::setServerFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]); } } if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) { if (!$DBServer->remoteIp) { $update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer); } } if ($DBServer->platform == \SERVER_PLATFORMS::EC2) { try { $info = $p->GetServerExtendedInformation($DBServer); $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : ''; } catch (\Exception $e) { // no need to do anything here; } $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus); } $update['status'] = \SERVER_STATUS::RESUMING; $update['dateAdded'] = date("Y-m-d H:i:s"); $DBServer->update($update); unset($update); continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) { $elasticIpAssigned = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING) { if (!$DBServer->remoteIp && !$DBServer->localIp) { $ipaddresses = $p->GetServerIPAddresses($DBServer); $elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer); if ($elasticIpAddress) { $ipaddresses['remoteIp'] = $elasticIpAddress; $DBServer->remoteIp = $elasticIpAddress; $elasticIpAssigned = true; } if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) { $DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]); } //Add tags Ec2Helper::createObjectTags($DBServer); } } } if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) { if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { // Check scalarizr deployment status $env = $DBServer->GetEnvironmentObject(); $azure = $env->azure(); $info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME)); $extensions = !empty($info->extensions) ? $info->extensions : []; foreach ($extensions as $extension) { /* @var $extension ExtensionData */ if ($extension->name == 'scalarizr') { $extStatus = $extension->statuses[0]; /* @var $extStatus StatusData */ if ($extStatus->level == 'Error') { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]); } } } } } else { AzureHelper::setupScalrAgent($DBServer); } } try { if ($DBServer->isOpenstack()) { OpenstackHelper::setServerFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { \Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId)); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scriptingEvent = false; $eventName = null; if ($DBServer->status == SERVER_STATUS::PENDING) { $eventName = "hostInit"; $scriptingEvent = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $eventName = "hostUp"; $scriptingEvent = EVENT_TYPE::HOST_UP; } if ($scriptingEvent && $dtadded) { $hasPendingMessages = !!$db->GetOne("\n SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n ", [$DBServer->serverId]); $scriptingTimeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scriptingEvent, $DBServer->farmRoleId]); if ($scriptingTimeout) { $launchTimeout = $launchTimeout + $scriptingTimeout; } if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId)); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false); } catch (Exception $err) { $this->getLogger()->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } //Whether IP address is changed if (!$DBServer->IsRebooting() && !$elasticIpAssigned) { $ipaddresses = $p->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId)); \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO Check health } } } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) { //TODO: Terminated outside scalr while in SUSPENDED state $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = $p->GetServerIPAddresses($DBServer); // Private IP cannot be removed (only changed). if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null; if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform); $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() > $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO Check reboot timeout } } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } if (stristr($e->getMessage(), "not found")) { $this->getLogger()->fatal($e->getMessage()); } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(5); $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage()); } else { $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage()); } } } $this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime); return $request; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $dtNow = new DateTime('now'); try { $dbServer = DBServer::LoadByID($request->serverId); } catch (ServerNotFoundException $e) { $this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage()); return false; } if (!in_array($dbServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) { return false; } //Skip Locked instances if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) { return false; } //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if (in_array($dbServer->status, array(SERVER_STATUS::TERMINATED)) || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) { try { $p = PlatformFactory::NewPlatform($dbServer->platform); $environment = $dbServer->GetEnvironmentObject(); if (!$environment->isPlatformEnabled($dbServer->platform)) { throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id)); } if ($dbServer->GetCloudServerID()) { $serverHistory = $dbServer->getServerHistory(); $isTermination = in_array($dbServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)); $isSuspension = in_array($dbServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND)); $status = $dbServer->GetRealStatus(); if ($dbServer->isCloudstack()) { //Workaround for when expunge flag not working and servers stuck in Destroyed state. $isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed'; } else { $isTerminated = $status->isTerminated(); } if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) { try { if ($dbServer->farmId != 0) { try { if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]])); if ($dbServer->index == 1 && $serversCount > 1) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. " . "Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); return false; } } } catch (Exception $e) { } Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); } } catch (Exception $e) { $this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage()); } $terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME); if (!$terminationTime || time() - $terminationTime > 180) { if ($isTermination) { $p->TerminateServer($dbServer); } else { $p->SuspendServer($dbServer); } $dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time()); if ($dbServer->farmId) { $wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown')); if (!$wasHostDownFired) { $event = new HostDownEvent($dbServer); $event->isSuspended = !$isTermination; \Scalr::FireEvent($dbServer->farmId, $event); } } } } else { if ($dbServer->status == SERVER_STATUS::TERMINATED) { if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) { $errorResolution = true; $serverHistory->setTerminated(); $dbServer->Remove(); } } else { if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) { $dbServer->status = SERVER_STATUS::TERMINATED; $dbServer->Save(); $errorResolution = true; } else { if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) { $dbServer->status = SERVER_STATUS::SUSPENDED; $dbServer->remoteIp = ''; $dbServer->localIp = ''; $dbServer->Save(); $errorResolution = true; } } } if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) { //Automatic error resolution $ste->delete(); } } } else { //If there is no cloudserverID we don't need to add this server into server history. //$serverHistory->setTerminated(); $dbServer->Remove(); } } catch (InstanceNotFound $e) { if ($serverHistory) { $serverHistory->setTerminated(); } $dbServer->Remove(); } catch (Exception $e) { if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || stripos($e->getMessage(), "The request you have made requires authentication.") !== false || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) { //Postpones unsuccessful task for 30 minutes. $ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage()); $minutes = rand(30, 40); $ste->retryAfter = new \DateTime('+' . $minutes . ' minutes'); if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { //We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts $dbServer->Remove(); } $ste->save(); } else { $this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage()); throw $e; } } } return $request; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $db = \Scalr::getDb(); //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); $bundleTask = BundleTask::LoadById($request->bundleTaskId); if (!$bundleTask instanceof BundleTask) { $this->getLogger()->fatal("Could not load bundle task id: %s", $request->bundleTaskId); return false; } else { $this->bundleTask = $bundleTask; $this->getLogger()->info("Processing bundle task id: %d status: %s serverid: %s", $bundleTask->id, $bundleTask->status, $bundleTask->serverId); } try { $dbServer = DBServer::LoadByID($bundleTask->serverId); } catch (\Scalr\Exception\ServerNotFoundException $e) { if (!$bundleTask->snapshotId) { $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED; $bundleTask->setDate('finished'); $bundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $bundleTask->serverId); $bundleTask->Save(); return; } $this->getLogger()->warn("Could not load server: %s. %s says: %s", $bundleTask->serverId, get_class($e), $e->getMessage()); } catch (Exception $e) { $this->getLogger()->warn("Could not load server: %s. %s says: %s", $bundleTask->serverId, get_class($e), $e->getMessage()); } switch ($bundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION: $conn = @fsockopen($dbServer->getSzrHost(), $dbServer->getPort(DBServer::PORT_CTRL), $errno, $errstr, 10); if ($conn) { $dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1); $this->bundleTaskLog("Outbound connection successfully established. Awaiting user action: prebuild automation selection"); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION; $this->bundleTaskLog(sprintf(_("Bundle task status: %s"), $bundleTask->status)); $bundleTask->Save(); } else { $errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $dbServer->getSzrHost(), $dbServer->getPort(DBServer::PORT_CTRL), $errstr); $errMsg = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR); if (!$errMsg || $errstr != $errMsg) { $dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr); $this->bundleTaskLog("{$errstr} Will try again in a few minutes."); } } return false; case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION: //nothing to do; return false; case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $bundleTask->setDate('started'); case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: if (!PlatformFactory::NewPlatform($dbServer->platform)->GetServerID($dbServer)) { $this->bundleTaskLog(sprintf(_("Waiting for temporary server"))); return false; } $status = PlatformFactory::NewPlatform($dbServer->platform)->GetServerRealStatus($dbServer); if ($status->isPending()) { //Server is in pensing state $this->bundleTaskLog(sprintf(_("Server status: %s"), $status->getName())); $this->bundleTaskLog(sprintf(_("Waiting for running state."), $status->getName())); return false; } elseif ($status->isTerminated()) { $this->bundleTaskLog(sprintf(_("Server status: %s"), $status->getName())); $dbServer->status = SERVER_STATUS::TERMINATED; $dbServer->save(); $bundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud."); return false; } break; } $this->getLogger()->info("Continue bundle task id:%d status:%s", $bundleTask->id, $bundleTask->status); switch ($bundleTask->status) { case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER: $ips = PlatformFactory::NewPlatform($dbServer->platform)->GetServerIPAddresses($dbServer); $dbServer->remoteIp = $ips['remoteIp']; $dbServer->localIp = $ips['localIp']; $dbServer->save(); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV; $bundleTask->save(); $this->bundleTaskLog(sprintf(_("Bundle task status: %s"), $bundleTask->status)); break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV: $this->bundleTaskLog(sprintf(_("Initializing SSH2 session to the server"))); if ($dbServer->platform == SERVER_PLATFORMS::IDCF && !$dbServer->remoteIp) { try { $this->bundleTaskLog("Creating port forwarding rules to be able to connect to the server by SSH"); $environment = $dbServer->GetEnvironmentObject(); $cloudLocation = $dbServer->GetCloudLocation(); $platform = PlatformFactory::NewPlatform($dbServer->platform); $sharedIpId = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP_ID . ".{$cloudLocation}", $environment, false); $sharedIp = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP . ".{$cloudLocation}", $environment, false); $this->bundleTaskLog("Shared IP: {$sharedIp}"); $cs = $environment->cloudstack($dbServer->platform); // Create port forwarding rules for scalarizr $port = $platform->getConfigVariable(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false); if (!$port) { $port1 = 30000; $port2 = 30001; $port3 = 30002; $port4 = 30003; } else { $port1 = $port + 1; $port2 = $port1 + 1; $port3 = $port2 + 1; $port4 = $port3 + 1; } $virtualmachineid = $dbServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID); $result2 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8014, 'protocol' => "udp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid)); $result1 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8013, 'protocol' => "tcp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid)); $result3 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8010, 'protocol' => "tcp", 'publicport' => $port3, 'virtualmachineid' => $virtualmachineid)); $result4 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8008, 'protocol' => "tcp", 'publicport' => $port2, 'virtualmachineid' => $virtualmachineid)); $result5 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 22, 'protocol' => "tcp", 'publicport' => $port4, 'virtualmachineid' => $virtualmachineid)); $dbServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4)); $dbServer->remoteIp = $sharedIp; $dbServer->Save(); $platform->setConfigVariable(array(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false); } catch (Exception $e) { $this->bundleTaskLog("Unable to create port-forwarding rules: {$e->getMessage()}"); } return false; } if ($dbServer->platform == SERVER_PLATFORMS::ECS && !$dbServer->remoteIp) { $this->bundleTaskLog(sprintf(_("Server doesn't have public IP. Assigning..."))); $osClient = $dbServer->GetEnvironmentObject()->openstack($dbServer->platform, $dbServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION)); $ports = $osClient->network->ports->list(); foreach ($ports as $port) { if ($port->device_id == $dbServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $serverNetworkPort = $port->id; break; } } $ips = $osClient->network->floatingIps->list(); //Check free existing IP $ipAssigned = false; $ipAddress = false; $ipId = false; $ipInfo = false; foreach ($ips as $ip) { if ($ip->port_id && $ip->port_id == $serverNetworkPort) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipAssigned = true; $ipInfo = $ip; break; } if (!$ip->fixed_ip_address && !$ipAddress) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipInfo = $ip; } } if (!$ipAssigned) { if (!$serverNetworkPort) { $this->bundleTaskLog("Unable to identify network port of instance"); return false; } else { if (!$ipAddress) { $networks = $osClient->network->listNetworks(); foreach ($networks as $network) { if ($network->{"router:external"} == true) { $publicNetworkId = $network->id; } } if (!$publicNetworkId) { $this->bundleTaskLog("Unable to identify public network to allocate"); return false; } else { $ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort); $ipAddress = $ip->floating_ip_address; $dbServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); $this->bundleTaskLog("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}"); } } else { $this->bundleTaskLog("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")"); $osClient->network->floatingIps->update($ipId, $serverNetworkPort); } } } else { $this->bundleTaskLog("IP: {$ipAddress} already assigned"); } if ($ipAddress) { $dbServer->remoteIp = $ipAddress; $dbServer->Save(); } return false; } try { $ssh2Client = $dbServer->GetSsh2Client(); $ssh2Client->connect($dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $this->bundleTaskLog(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed return false; } $this->bundleTaskLog(sprintf(_("Created SSH session. Username: %s"), $ssh2Client->getLogin())); //Prepare script $this->bundleTaskLog(sprintf(_("Uploading builder scripts..."))); $behaviors = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR); try { if ($dbServer->isOpenstack()) { $platform = SERVER_PLATFORMS::OPENSTACK; } else { $platform = $dbServer->platform; } $baseUrl = rtrim(\Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host'), '/'); $options = array('server-id' => $dbServer->serverId, 'role-name' => $bundleTask->roleName, 'crypto-key' => $dbServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $dbServer->envId, 'region' => $dbServer->GetCloudLocation(), 'scalr-id' => SCALR_ID); $command = 'scalarizr --import -y'; foreach ($options as $k => $v) { $command .= sprintf(' -o %s=%s', $k, $v); } if ($dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') { $recipes = 'mysql=percona'; } else { $recipes = ''; } $scalarizrBranch = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH); $scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl"); /* %CHEF_SERVER_URL% %CHEF_VALIDATOR_NAME% %CHEF_VALIDATOR_KEY% %CHEF_ENVIRONMENT% %CHEF_ROLE_NAME% */ $chefServerId = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID); if ($chefServerId) { $chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId)); $chefServerInfo['v_auth_key'] = \Scalr::getContainer()->crypto->decrypt($chefServerInfo['v_auth_key']); } $scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents); if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) { throw new Exception("Cannot upload script"); } /* $this->bundleTaskLog(sprintf(_("Uploading chef recipes..."))); if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) { throw new Exception("Cannot upload chef recipes"); } */ } catch (Exception $e) { $this->bundleTaskLog(sprintf(_("Scripts upload failed: %s"), $e->getMessage())); //TODO: Set status of bundle log to failed return false; } $this->bundleTaskLog("Launching role builder routines on server"); $ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh"); // For CGE we need to use sudo if ($bundleTask->platform == SERVER_PLATFORMS::GCE || $bundleTask->osFamily == 'amazon') { $shell = $ssh2Client->getShell(); @stream_set_blocking($shell, true); @stream_set_timeout($shell, 5); @fwrite($shell, "sudo touch /var/log/role-builder-output.log 2>&1" . PHP_EOL); $output = @fgets($shell, 4096); $this->bundleTaskLog("Verbose 1: {$output}"); @fwrite($shell, "sudo chmod 0666 /var/log/role-builder-output.log 2>&1" . PHP_EOL); $output2 = @fgets($shell, 4096); $this->bundleTaskLog("Verbose 2: {$output2}"); @fwrite($shell, "sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &" . PHP_EOL); $output3 = @fgets($shell, 4096); $this->bundleTaskLog("Verbose 3: {$output3}"); sleep(5); $meta = stream_get_meta_data($shell); $this->bundleTaskLog(sprintf("Verbose (Meta): %s", json_encode($meta))); $i = 4; if ($meta['eof'] == false && $meta['unread_bytes'] != 0) { $output4 = @fread($shell, $meta['unread_bytes']); $this->bundleTaskLog("Verbose {$i}: {$output4}"); $meta = stream_get_meta_data($shell); $this->bundleTaskLog(sprintf("Verbose (Meta): %s", json_encode($meta))); } @fclose($shell); /* $r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log"); $this->bundleTaskLog("1: {$r1} ({$ssh2Client->stdErr})"); $r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log"); $this->bundleTaskLog("2: {$r2} ({$ssh2Client->stdErr})"); $r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); $this->bundleTaskLog("3: {$r3} ({$ssh2Client->stdErr})"); */ } else { $ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &"); } $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE; $bundleTask->save(); break; case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE: try { $ssh2Client = $dbServer->GetSsh2Client(); $ssh2Client->connect($dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH)); } catch (Exception $e) { $this->bundleTaskLog(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH), $e->getMessage())); //TODO: Set status of bundle log to failed return false; } $log = $ssh2Client->getFile('/var/log/role-builder-output.log'); $log_lines = explode("\r\n", $log); $last_msg = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE); while ($msg = trim(array_shift($log_lines))) { if (substr($msg, -1, 1) != ']') { continue; } if ($last_msg) { if ($msg != $last_msg) { continue; } elseif ($msg == $last_msg) { $last_msg = null; continue; } } if (stristr($msg, '[ Failed ]')) { $stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log'); $this->bundleTaskLog(sprintf("role-builder-step.log: %s", $stepLog)); $bundleTask->SnapshotCreationFailed($msg); } else { $this->bundleTaskLog($msg); $dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg); } } //Read /var/log/role-builder-output.log break; case SERVER_SNAPSHOT_CREATION_STATUS::PENDING: try { $platformModule = PlatformFactory::NewPlatform($bundleTask->platform); $platformModule->CreateServerSnapshot($bundleTask); } catch (Exception $e) { $this->getLogger()->error($e->getMessage()); $bundleTask->SnapshotCreationFailed($e->getMessage()); } break; case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING: $addedTime = strtotime($bundleTask->dateAdded); if ($addedTime + 3600 < time()) { $bundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time."); } break; case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS: PlatformFactory::NewPlatform($bundleTask->platform)->CheckServerSnapshotStatus($bundleTask); break; case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE: try { if ($bundleTask->object == BundleTask::BUNDLETASK_OBJECT_IMAGE) { if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $dbRole = $dbServer->GetFarmRoleObject()->GetRoleObject(); $dbRole->__getNewRoleObject()->setImage($bundleTask->platform, $bundleTask->cloudLocation, $bundleTask->snapshotId, $bundleTask->createdById, $bundleTask->createdByEmail); $this->bundleTaskLog(sprintf(_("Image replacement completed."))); } $this->bundleTaskLog(sprintf(_("Bundle task completed."))); $bundleTask->setDate('finished'); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $bundleTask->Save(); } elseif ($bundleTask->object == BundleTask::BUNDLETASK_OBJECT_ROLE) { if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $saveOldRole = false; try { $dbRole = $dbServer->GetFarmRoleObject()->GetRoleObject(); if ($dbRole->name == $bundleTask->roleName && $dbRole->envId == $bundleTask->envId) { $saveOldRole = true; } } catch (Exception $e) { //NO OLD ROLE } if ($dbRole && $saveOldRole) { if ($dbServer) { $new_role_name = BundleTask::GenerateRoleName($dbServer->GetFarmRoleObject(), $dbServer); } else { $new_role_name = $bundleTask->roleName . "-" . rand(1000, 9999); } $dbRole->name = $new_role_name; $this->bundleTaskLog(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $bundleTask->roleName, $dbRole->id, $new_role_name)); $dbRole->save(); } } try { $DBRole = DBRole::createFromBundleTask($bundleTask); } catch (Exception $e) { $bundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again."); return; } if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) { $bundleTask->setDate('finished'); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $this->bundleTaskLog(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS)); } else { try { $this->bundleTaskLog(sprintf(_("Replacement type: %s"), $bundleTask->replaceType)); $r_farm_roles = array(); try { $DBFarm = DBFarm::LoadByID($bundleTask->farmId); } catch (Exception $e) { if (stristr($e->getMessage(), "not found in database")) { $bundleTask->SnapshotCreationFailed("Farm was removed before task was finished"); } return; } if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) { try { $r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($bundleTask->prototypeRoleId); } catch (Exception $e) { } } elseif ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) { $farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($bundleTask->prototypeRoleId, $bundleTask->envId)); foreach ($farm_roles as $farm_role) { try { $r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']); } catch (Exception $e) { } } } foreach ($r_farm_roles as $DBFarmRole) { if ($DBFarmRole->CloudLocation != $bundleTask->cloudLocation) { $this->bundleTaskLog(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID)); } else { $DBFarmRole->RoleID = $bundleTask->roleId; $DBFarmRole->Save(); } } $this->bundleTaskLog(sprintf(_("Replacement completed. Bundle task completed."))); try { if ($dbServer->status == SERVER_STATUS::IMPORTING) { $dbServer->Remove(); } elseif ($dbServer->status == SERVER_STATUS::TEMPORARY) { $this->bundleTaskLog("Terminating temporary server"); $dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); $this->bundleTaskLog("Termination request has been sent"); } } catch (Exception $e) { $this->bundleTaskLog("Warning: {$e->getMessage()}"); } $bundleTask->setDate('finished'); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; $bundleTask->Save(); } catch (Exception $e) { $this->getLogger()->error($e->getMessage()); $this->bundleTaskLog(sprintf(_("Server replacement failed: %s"), $e->getMessage())); $bundleTask->setDate('finished'); $bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS; } } } if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) { try { if ($dbServer->status == SERVER_STATUS::IMPORTING) { $dbServer->Remove(); } elseif ($dbServer->status == SERVER_STATUS::TEMPORARY) { $this->bundleTaskLog("Terminating temporary server"); $dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER); $this->bundleTaskLog("Termination request has been sent"); } } catch (Exception $e) { $this->bundleTaskLog("Warning: {$e->getMessage()}"); } } $bundleTask->Save(); } catch (Exception $e) { $this->getLogger()->error($e->getMessage()); } break; } return $request; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if (!isset($request->farmRoleId)) { //This is the farm with synchronous launch of roles try { $DBFarm = DBFarm::LoadByID($request->farmId); if ($DBFarm->Status != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID); return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId); throw $e; } //Gets the list of the roles $list = $DBFarm->GetFarmRoles(); } else { //This is asynchronous lauhch try { $DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId); if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) { //We don't need to handle inactive farms return false; } } catch (Exception $e) { $this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId); throw $e; } $list = [$DBFarmRole]; } $this->getLogger()->debug("Processing %s FarmRoles", count($list)); foreach ($list as $DBFarmRole) { // Set Last polling time $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL); $disabledScaling = false; if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') { if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) { // For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running) // When 1 instance is already running, the rest is fully manual $roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount(); if ($roleTotalInstances != 0) { $disabledScaling = true; } } else { if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { $disabledScaling = true; } } if ($disabledScaling) { $this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias); continue; } } $farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name; // Get current count of running and pending instances. $this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName)); $scalingManager = new Scalr_Scaling_Manager($DBFarmRole); //Replacing the logger $scalingManager->logger = $this->getLogger(); $scalingDecision = $scalingManager->makeScalingDecision(); $scalingDecisionDetails = $scalingManager->decisonInfo; $this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails)); if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) { return; } if ($scalingDecision == Scalr_Scaling_Decision::NOOP) { continue; } else { if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) { /* Timeout instance's count decrease. Decreases instances count after scaling resolution the spare instances are running for selected timeout interval from scaling EditOptions */ // We have to check timeout limits before new scaling (downscaling) process will be initiated if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_down_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null)); continue; } } // end Timeout instance's count decrease $sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC'; $servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID)); $got_valid_instance = false; $ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR); $useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN); $isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ); // Select instance that will be terminated // // Instances ordered by uptime (oldest wil be choosen) // Instance cannot be mysql master // Choose the one that was rebundled recently $DBServer = null; while (!$got_valid_instance && count($servers) > 0) { $item = array_shift($servers); $DBServer = DBServer::LoadByID($item['server_id']); if ($isRabbitMQ) { $serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]); if ($DBServer->index == 1 && $serverExists) { continue; } } if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) { continue; } // Exclude db master if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) { $got_valid_instance = true; } //Check safe shutdown if ($useSafeShutdown == 1) { try { $res = $DBServer->scalarizr->system->callAuthShutdownHook(); } catch (Exception $e) { $res = $e->getMessage(); } if ($res != '1') { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); $got_valid_instance = false; } } } // end while if ($DBServer !== null && $got_valid_instance) { $this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId)); $allow_terminate = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $aws = $DBServer->GetEnvironmentObject()->aws($DBServer); // Shutdown an instance just before a full hour running if (!$ignoreFullHour) { $response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0); if ($response && count($response->instancesSet)) { $launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp(); $time = 3600 - (time() - $launch_time) % 3600; // Terminate instance in < 10 minutes for full hour. if ($time <= 600) { $allow_terminate = true; } else { $timeout = round(($time - 600) / 60, 1); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } } else { $allow_terminate = true; } //Releases memory $DBServer->GetEnvironmentObject()->getContainer()->release('aws'); unset($aws); } else { $allow_terminate = true; } if ($allow_terminate) { $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } try { if ($terminateStrategy == 'terminate') { $DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } else { $DBServer->suspend('SCALING_DOWN', false); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } } catch (Exception $e) { $this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId)); } } } else { $this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID)); } //break; } elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) { /* Timeout instance's count increase. Increases instance's count after scaling resolution 'need more instances' for selected timeout interval from scaling EditOptions */ if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) { // if the farm timeout is exceeded // checking timeout interval. $last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME); $timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT); // check the time interval to continue scaling or cancel it... if (time() - $last_up_scale_data_time < $timeout_interval * 60) { // if the launch time is too small to terminate smth in this role -> go to the next role in foreach() \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias)); continue; } } // end Timeout instance's count increase //Check DBMsr. Do not start slave during slave2master process $isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior(); if ($isDbMsr) { if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) { $runningServers = $DBFarmRole->GetRunningInstancesCount(); if ($runningServers > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves"))); continue; } else { $DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL); } } } if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) { $pendingInstances = $DBFarmRole->GetPendingInstancesCount(); if ($pendingInstances > 0) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName))); continue; } } $fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId)); if ($fstatus != FARM_STATUS::RUNNING) { $this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it."); return; } $terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY); if (!$terminateStrategy) { $terminateStrategy = 'terminate'; } $suspendedServer = null; if ($terminateStrategy == 'suspend') { $suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED)); if (count($suspendedServers) > 0) { $suspendedServer = array_shift($suspendedServers); } } if ($terminateStrategy == 'suspend' && $suspendedServer) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId)); } if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) { $ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole); try { $DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP); $DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId)); } catch (Exception $e) { \Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage()); } } else { $platform = PlatformFactory::NewPlatform($suspendedServer->platform); $platform->ResumeServer($suspendedServer); } } } } return true; }
public function StartThread($mysql_farm_role) { // Reconfigure observers; Scalr::ReconfigureObservers(); $db = \Scalr::getDb(); $DBFarmRole = DBFarmRole::LoadByID($mysql_farm_role['id']); try { $DBFarm = $DBFarmRole->GetFarmObject(); } catch (Exception $e) { return; } //skip terminated farms if ($DBFarm->Status != FARM_STATUS::RUNNING) { return; } $tz = Scalr_Environment::init()->loadById($DBFarm->EnvID)->getPlatformConfigValue(ENVIRONMENT_SETTINGS::TIMEZONE); if ($tz) { date_default_timezone_set($tz); } // // Check replication status // $this->Logger->info("[FarmID: {$DBFarm->ID}] Checking replication status"); $servers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::RUNNING)); // // Check backups and mysql bandle procedures // //Backups if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_ENABLED) && $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) != 0) { if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING) == 1) { // Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots) // We must wait for running bundle process. $bcp_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) * 60 * 5; if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BCP_TS) + $bcp_timeout < time()) { $bcp_timeouted = true; } if ($bcp_timeouted) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING, 0, DBFarmRole::TYPE_LCL); $this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Backup already running. Timeout. Clear lock."); } } else { $timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) * 60; if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BCP_TS) + $timeout < time()) { $this->Logger->info("[FarmID: {$DBFarm->ID}] Need new backup"); $servers = $DBFarm->GetMySQLInstances(false, true); if (!$servers[0]) { $servers = $DBFarm->GetMySQLInstances(true); } else { $servers = array_reverse($servers); } $DBServer = $servers[0]; if ($DBServer) { if ($DBServer->status == SERVER_STATUS::RUNNING) { $msg = new Scalr_Messaging_Msg_Mysql_CreateBackup($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_ROOT_PASSWORD)); $DBServer->SendMessage($msg); $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING, 1, DBFarmRole::TYPE_LCL); $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_BCP_SERVER_ID, $DBServer->serverId, DBFarmRole::TYPE_LCL); } } else { $this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql instances for run backup procedure!"); } } } } if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_ENABLED) && $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY) != 0) { if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING) == 1) { // Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots) // We must wait for running bundle process. $bundle_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY) * (3600 * 2); if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BUNDLE_TS) + $bundle_timeout < time()) { $bundle_timeouted = true; } if ($bundle_timeouted) { $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING, 0, DBFarmRole::TYPE_LCL); $this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Bundle already running. Timeout. Clear lock."); } } else { /* * Check bundle window */ $bundleEvery = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY); $timeout = $bundleEvery * 3600; $lastBundleTime = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BUNDLE_TS); $performBundle = false; if ($bundleEvery % 24 == 0) { if ($lastBundleTime) { $days = $bundleEvery / 24; $bundleDay = (int) date("md", strtotime("+{$days} day", $lastBundleTime)); if ($bundleDay > (int) date("md")) { return; } } $pbwFrom = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_MM)); $pbwTo = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_MM)); if ($pbwFrom && $pbwTo) { $current_time = (int) date("Hi"); if ($pbwFrom <= $current_time && $pbwTo >= $current_time) { $performBundle = true; } } else { $performBundle = true; } } else { //Check timeout if ($lastBundleTime + $timeout < time()) { $pbwFrom = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_MM)); $pbwTo = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_MM)); if ($pbwFrom && $pbwTo) { $current_time = (int) date("Hi"); if ($pbwFrom <= $current_time && $pbwTo >= $current_time) { $performBundle = true; } } else { $performBundle = true; } } } if ($performBundle) { $this->Logger->info("[FarmID: {$DBFarm->ID}] Need mySQL bundle procedure"); // Rebundle $servers = $DBFarm->GetMySQLInstances(true, false); $DBServer = $servers[0]; if ($DBServer) { if ($DBServer->status == SERVER_STATUS::RUNNING) { $DBServer->SendMessage(new Scalr_Messaging_Msg_Mysql_CreateDataBundle()); $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING, 1, DBFarmRole::TYPE_LCL); $DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_SERVER_ID, $DBServer->serverId, DBFarmRole::TYPE_LCL); } } else { $this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql master instances for run bundle procedure!"); } } } } // Check replication foreach ($servers as $DBServer) { try { if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) == 1) { continue; } $this->Logger->info("[FarmID: {$DBFarm->ID}] {$DBServer->remoteIp} -> SLAVE STATUS"); $sock = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP); @socket_set_nonblock($sock); $time = time(); $res = true; while (!@socket_connect($sock, $DBServer->remoteIp, 3306)) { $err = @socket_last_error($sock); if ($err == 115 || $err == 114 || $err == 36 || $err == 37) { if (time() - $time >= 5) { @socket_close($sock); $res = false; break; } sleep(1); continue; } else { $res = $err == 56 ? true : false; break; } } if (!$res) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Scalr cannot connect to server %s:3306 (%s) and check replication status. (Error (%s):%s)"), $DBServer->remoteIp, $DBServer->serverId, $err, socket_strerror($err)))); continue; } // Connect to Mysql on slave $conn = NewADOConnection("mysqli"); $conn->Connect($DBServer->remoteIp, 'scalr_stat', $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_MYSQL_STAT_PASSWORD), null); $conn->SetFetchMode(ADODB_FETCH_ASSOC); // Get Slave status $r = $conn->GetRow("SHOW SLAVE STATUS"); // Check slave replication running or not if ($r['Slave_IO_Running'] == 'Yes' && $r['Slave_SQL_Running'] == 'Yes') { $replication_status = 1; } else { $replication_status = 0; } if ($replication_status != $DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_REPLICATION_STATUS)) { if ($replication_status == 0) { Scalr::FireEvent($DBFarm->ID, new MySQLReplicationFailEvent($DBServer)); } else { Scalr::FireEvent($DBFarm->ID, new MySQLReplicationRecoveredEvent($DBServer)); } } } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, "Cannot retrieve replication status. {$e->getMessage()}")); } } }