Method for multiprocess scripts. We must recreate DB connection created in constructor
| public static ReconfigureObservers ( ) |
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::startChild()
*/
function startChild()
{
// Reopen DB connection in child
$this->db = $this->getContainer()->adodb;
// Reconfigure observers;
Scalr::ReconfigureObservers();
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
try {
$dbFarm = DBFarm::LoadByID($request->farmId);
$curDate = new DateTime();
$tdValue = $dbFarm->GetSetting(DBFarm::SETTING_LEASE_TERMINATE_DATE);
if ($tdValue) {
$td = new DateTime($tdValue);
if ($td < $curDate) {
//Terminates farm
$event = new FarmTerminatedEvent(0, 1, false, 1);
SettingEntity::increase(SettingEntity::LEASE_TERMINATE_FARM);
\Scalr::FireEvent($request->farmId, $event);
$this->log('INFO', sprintf('Farm: %s [ID: %d] was terminated by lease manager', $dbFarm->Name, $dbFarm->ID));
} else {
// only inform user
$days = $td->diff($curDate)->days;
$notifications = json_decode($dbFarm->GetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND), true);
$governance = new Scalr_Governance($dbFarm->EnvID);
$settings = $governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'notifications');
if (is_array($settings)) {
foreach ($settings as $n) {
if (!$notifications[$n['key']] && $n['period'] >= $days) {
$mailer = \Scalr::getContainer()->mailer;
$tdHuman = Scalr_Util_DateTime::convertDateTime($td, $dbFarm->GetSetting(DBFarm::SETTING_TIMEZONE), 'M j, Y');
if ($n['to'] == 'owner') {
$user = new Scalr_Account_User();
$user->loadById($dbFarm->createdByUserId);
if (\Scalr::config('scalr.auth_mode') == 'ldap') {
$email = $user->getSetting(Scalr_Account_User::SETTING_LDAP_EMAIL);
if (!$email) {
$email = $user->getEmail();
}
} else {
$email = $user->getEmail();
}
$mailer->addTo($email);
} else {
foreach (explode(',', $n['emails']) as $email) {
$mailer->addTo(trim($email));
}
}
$mailer->sendTemplate(SCALR_TEMPLATES_PATH . '/emails/farm_lease_terminate.eml', array('{{terminate_date}}' => $tdHuman, '{{farm}}' => $dbFarm->Name, '{{envName}}' => $dbFarm->GetEnvironmentObject()->name, '{{envId}}' => $dbFarm->GetEnvironmentObject()->id));
$notifications[$n['key']] = 1;
$dbFarm->SetSetting(DBFarm::SETTING_LEASE_NOTIFICATION_SEND, json_encode($notifications));
$this->log('INFO', "Notification was sent by key: %s about farm: %s [ID: %d] by lease manager", $n['key'], $dbFarm->Name, $dbFarm->ID);
}
}
}
}
}
} catch (Exception $e) {
throw $e;
}
return $request;
}
function startChild()
{
// Reopen DB connection in child
$this->db = Core::GetDBInstance(null, true);
// Reconfigure observers;
Scalr::ReconfigureObservers();
}
function startChild()
{
// Reopen DB connection in child
$this->db = Core::GetDBInstance(null, true);
// Reconfigure observers;
Scalr::ReconfigureObservers();
$this->snmpWatcher = new SNMPWatcher();
foreach (array_keys($this->watchers) as $watcher_name) {
$this->snmpWatcher->SetOIDs($watcher_name);
}
}
public function StartThread($data)
{
// Reconfigure observers;
Scalr::ReconfigureObservers();
//
// Create pid file
//
@file_put_contents(CACHEPATH . "/" . __CLASS__ . ".Daemon.pid", posix_getpid());
// Get memory usage on start
$memory_usage = $this->GetMemoryUsage();
$this->Logger->info("SzrMessaging daemon started. Memory usage: {$memory_usage}M");
// Get DB instance
$db = Core::GetDBInstance();
$FarmObservers = array();
$job = new Scalr_Cronjob_ScalarizrMessaging();
while (true) {
$rows = $db->Execute("SELECT distinct(server_id) FROM messages WHERE type = ? AND status = ? AND isszr = ?", array("in", MESSAGE_STATUS::PENDING, 1));
while ($row = $rows->FetchRow()) {
try {
$serverId = $row['server_id'];
$job->handleWork($serverId);
} catch (Exception $e) {
}
}
// Cleaning
unset($current_memory_usage);
unset($event);
// Check memory usage
$current_memory_usage = $this->GetMemoryUsage() - $memory_usage;
if ($current_memory_usage > $this->DaemonMemoryLimit) {
$this->Logger->warn("SzrMessaging daemon reached memory limit {$this->DaemonMemoryLimit}M, Used:{$current_memory_usage}M");
$this->Logger->warn("Restart daemon.");
exit;
}
// Sleep for 60 seconds
sleep(5);
// Clear stat file cache
clearstatcache();
// Check daemon file for modifications.
if ($this->DaemonMtime && $this->DaemonMtime < @filemtime(__FILE__)) {
$this->Logger->warn(__FILE__ . " - updated. Exiting for daemon reload.");
exit;
}
}
}
public function StartThread($data)
{
// Reconfigure observers;
Scalr::ReconfigureObservers();
//
// Create pid file
//
@file_put_contents(CACHEPATH . "/" . __CLASS__ . ".Daemon.pid", posix_getpid());
// Get memory usage on start
$memory_usage = $this->GetMemoryUsage();
$this->Logger->info("DBQueueEventProcess daemon started. Memory usage: {$memory_usage}M");
// Get DB instance
$db = Core::GetDBInstance();
$FarmObservers = array();
while (true) {
// Process tasks from Deferred event queue
while ($Task = TaskQueue::Attach(QUEUE_NAME::DEFERRED_EVENTS)->Poll()) {
$Task->Run();
}
// Reset task
TaskQueue::Attach(QUEUE_NAME::DEFERRED_EVENTS)->Reset();
// Cleaning
unset($current_memory_usage);
unset($event);
// Check memory usage
$current_memory_usage = $this->GetMemoryUsage() - $memory_usage;
if ($current_memory_usage > $this->DaemonMemoryLimit) {
$this->Logger->warn("DBQueueEventProcess daemon reached memory limit {$this->DaemonMemoryLimit}M, Used:{$current_memory_usage}M");
$this->Logger->warn("Restart daemon.");
exit;
}
// Sleep for 60 seconds
sleep(15);
// Clear stat file cache
clearstatcache();
// Check daemon file for modifications.
if ($this->DaemonMtime && $this->DaemonMtime < @filemtime(__FILE__)) {
$this->Logger->warn(__FILE__ . " - updated. Exiting for daemon reload.");
exit;
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$dtNow = new DateTime('now');
try {
$dbServer = DBServer::LoadByID($request->serverId);
} catch (ServerNotFoundException $e) {
$this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage());
return false;
}
if (!in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED])) {
return false;
}
// Check and skip locked instances
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) {
if (($checkDateTime = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME)) <= time()) {
if (!$dbServer->GetRealStatus(true)->isTerminated()) {
$isLocked = $dbServer->GetEnvironmentObject()->aws($dbServer->GetCloudLocation())->ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
if ($isLocked) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Server '%s' has disableAPITermination flag and can't be terminated (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
$startTime = strtotime($dbServer->dateShutdownScheduled);
// 1, 2, 3, 4, 5, 6, 9, 14, ... 60
$diff = round((($checkDateTime < $startTime ? $startTime : $checkDateTime) - $startTime) / 60 * 0.5) * 60;
$diff = $diff == 0 ? 60 : ($diff > 3600 ? 3600 : $diff);
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME, time() + $diff);
return false;
} else {
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isLocked);
}
}
} else {
return false;
}
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) {
try {
$p = PlatformFactory::NewPlatform($dbServer->platform);
$environment = $dbServer->GetEnvironmentObject();
if (!$environment->isPlatformEnabled($dbServer->platform)) {
throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id));
}
if ($dbServer->GetCloudServerID()) {
$serverHistory = $dbServer->getServerHistory();
$isTermination = in_array($dbServer->status, [SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE]);
$isSuspension = in_array($dbServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND]);
/* @var $terminationData TerminationData */
$terminationData = null;
//NOTE: in any case, after call, be sure to set callback to null
$this->setupClientCallback($p, function ($request, $response) use($dbServer, &$terminationData) {
$terminationData = new TerminationData();
$terminationData->serverId = $dbServer->serverId;
if ($request instanceof \http\Client\Request) {
$terminationData->requestUrl = $request->getRequestUrl();
$terminationData->requestQuery = $request->getQuery();
$terminationData->request = $request->toString();
}
if ($response instanceof \http\Client\Response) {
$terminationData->response = $response->toString();
$terminationData->responseCode = $response->getResponseCode();
$terminationData->responseStatus = $response->getResponseStatus();
}
}, $dbServer);
try {
$status = $dbServer->GetRealStatus();
} catch (Exception $e) {
//eliminate callback
$this->setupClientCallback($p, null, $dbServer);
throw $e;
}
//eliminate callback
$this->setupClientCallback($p, null, $dbServer);
if ($dbServer->isCloudstack()) {
//Workaround for when expunge flag not working and servers stuck in Destroyed state.
$isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed';
} else {
$isTerminated = $status->isTerminated();
}
if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) {
try {
if ($dbServer->farmId != 0) {
try {
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]]));
if ($dbServer->index == 1 && $serversCount > 1) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
return false;
}
}
} catch (Exception $e) {
}
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
}
} catch (Exception $e) {
$this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage());
}
$terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME);
if (!$terminationTime || time() - $terminationTime > 180) {
if ($isTermination) {
$p->TerminateServer($dbServer);
} else {
$p->SuspendServer($dbServer);
}
$dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time());
if ($dbServer->farmId) {
$wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbServer);
$event->isSuspended = !$isTermination;
\Scalr::FireEvent($dbServer->farmId, $event);
}
}
}
} else {
if ($dbServer->status == SERVER_STATUS::TERMINATED) {
if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) {
$errorResolution = true;
$serverHistory->setTerminated();
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
}
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) {
$dbServer->updateStatus(SERVER_STATUS::TERMINATED);
$errorResolution = true;
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) {
$dbServer->update(['status' => SERVER_STATUS::SUSPENDED, 'remoteIp' => '', 'localIp' => '']);
$errorResolution = true;
}
}
}
if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) {
//Automatic error resolution
$ste->delete();
}
}
} else {
//If there is no cloudserverID we don't need to add this server into server history.
//$serverHistory->setTerminated();
$dbServer->Remove();
}
} catch (InstanceNotFound $e) {
if ($serverHistory) {
$serverHistory->setTerminated();
}
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
} catch (Exception $e) {
if ($request->serverId && stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false && $dbServer && $dbServer->platform == \SERVER_PLATFORMS::EC2) {
// server has disableApiTermination flag on cloud, update server properties
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, 1);
} else {
if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || CloudPlatformSuspensionInfo::isSuspensionException($e) || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) {
//Postpones unsuccessful task for 30 minutes.
$ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage());
$minutes = rand(30, 40);
$ste->retryAfter = new \DateTime('+' . $minutes . ' minutes');
if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
//We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
}
$ste->save();
} else {
$this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage());
throw $e;
}
}
}
}
return $request;
}
public function StartThread($bundle_task_info)
{
$db = Core::GetDBInstance();
// Reconfigure observers;
Scalr::ReconfigureObservers();
$BundleTask = BundleTask::LoadById($bundle_task_info['id']);
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
} catch (ServerNotFoundException $e) {
if (!$BundleTask->snapshotId) {
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$BundleTask->setDate('finished');
$BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId);
$BundleTask->Save();
return;
}
} catch (Exception $e) {
//$this->Logger->error($e->getMessage());
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
// IF server is in pensing state
$status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer);
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
if ($status->isPending()) {
$BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName()));
exit;
} elseif ($status->isTerminated()) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
break;
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$DBServer->remoteIp = $ips['remoteIp'];
$DBServer->localIp = $ips['locateIp'];
$DBServer->save();
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$BundleTask->save();
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$BundleTask->Log(sprintf(_("Initializing SSH2 session to the server")));
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
//Prepare script
$BundleTask->Log(sprintf(_("Uploading builder scripts...")));
$behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
$options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $DBServer->platform, 'behaviour' => trim(str_replace("base", "", $behaviors)), 'queryenv-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/query-env", 'messaging-p2p.producer-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/messaging");
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_NODE_NAME%", "\r\n"), array($DBServer->platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', '', '', '', '', '', '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
$BundleTask->Log(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$BundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog));
$BundleTask->SnapshotCreationFailed($msg);
/** Terminate server **/
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$DBServer->status = SERVER_STATUS::PENDING_TERMINATE;
$DBServer->save();
$BundleTask->Log(sprintf("Temporary server '%s' (%s) has been terminated", $DBServer->serverId, $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)));
} else {
$BundleTask->Log($msg);
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($BundleTask->platform);
$platformModule->CreateServerSnapshot($BundleTask);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage());
$BundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($BundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS:
$r_farm_roles = array();
$BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType));
try {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$BundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND new_role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
$update_farm_dns_zones = array();
$completed_roles = 0;
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) {
$BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
$completed_roles++;
} else {
$servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
if (count($servers) == 0) {
$DBFarmRole->RoleID = $DBFarmRole->NewRoleID;
$DBFarmRole->NewRoleID = null;
$DBFarmRole->Save();
$update_farm_dns_zones[$DBFarmRole->FarmID] = 1;
$completed_roles++;
} else {
$metaData = $BundleTask->getSnapshotDetails();
foreach ($servers as $server) {
try {
$DBServer = DBServer::LoadByID($server['server_id']);
} catch (Exception $e) {
//TODO:
continue;
}
if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) {
$DBServer->roleId = $BundleTask->roleId;
$DBServer->Save();
if ($metaData['noServersReplace']) {
$BundleTask->Log(sprintf("'Do not replace servers' option was checked. Server '%s' won't be replaced to new image.", $DBServer->serverId));
} else {
$BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, already has all modifications. No need to replace it.", $DBServer->serverId));
}
if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t\t\t\t\t\t", array(sprintf("Farm was in 'Synchronizing' state. Server terminated when bundling was completed. Bundle task #%s", $BundleTask->id), $DBServer->serverId));
}
} else {
if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?)", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE))) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID);
$nDBServer = Scalr::LaunchServer($ServerCreateInfo);
$nDBServer->replaceServerID = $DBServer->serverId;
$nDBServer->Save();
$BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId));
}
}
// if serverid != bundletask->serverID
}
// foreach server
}
// count($servers)
}
}
if ($completed_roles == count($r_farm_roles)) {
$BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Save();
}
try {
if (count($update_farm_dns_zones) != 0) {
foreach ($update_farm_dns_zones as $farm_id => $v) {
$dnsZones = DBDNSZone::loadByFarmId($farm_id);
foreach ($dnsZones as $dnsZone) {
if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) {
$dnsZone->updateSystemRecords();
$dnsZone->save();
}
}
}
}
} catch (Exception $e) {
$this->Logger->fatal("DNS ZONE: {$e->getMessage()}");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = DBRole::loadById($DBServer->roleId);
if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($DBServer) {
$new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer);
} else {
$new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
} else {
//TODO:
//$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})");
}
}
try {
$DBRole = DBRole::createFromBundleTask($BundleTask);
} catch (Exception $e) {
$BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
if ($DBServer->farmId) {
// Create DBFarmRole object
// TODO: create DBFarm role
}
//$DBServer->Delete();
}
} catch (Exception $e) {
}
} else {
try {
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS));
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
$DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
} else {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
$DBFarmRole = DBFarmRole::LoadByID($farm_role['id']);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
}
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS;
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
$BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
$BundleTask->Save();
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
}
break;
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
try {
$dbFarm = DBFarm::LoadByID($request->farmId);
$curDate = new DateTime();
$tdValue = $dbFarm->GetSetting(Entity\FarmSetting::LEASE_TERMINATE_DATE);
if ($tdValue) {
$td = new DateTime($tdValue);
if ($td < $curDate) {
//Terminates farm
SettingEntity::increase(SettingEntity::LEASE_TERMINATE_FARM);
//Ajdusts both account & environment for the audit log
\Scalr::getContainer()->auditlogger->setAccountId($dbFarm->ClientID)->setAccountId($dbFarm->EnvID);
\Scalr::FireEvent($request->farmId, new FarmTerminatedEvent(0, 1, false, 1, true, null));
$this->log('INFO', sprintf('Farm: %s [ID: %d] was terminated by lease manager', $dbFarm->Name, $dbFarm->ID));
} else {
// only inform user
$days = $td->diff($curDate)->days;
$notifications = json_decode($dbFarm->GetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND), true);
$governance = new Scalr_Governance($dbFarm->EnvID);
$settings = $governance->getValue(Scalr_Governance::CATEGORY_GENERAL, Scalr_Governance::GENERAL_LEASE, 'notifications');
if (is_array($settings)) {
foreach ($settings as $n) {
if (!$notifications[$n['key']] && $n['period'] >= $days) {
$mailer = \Scalr::getContainer()->mailer;
$tdHuman = Scalr_Util_DateTime::convertDateTime($td, $dbFarm->GetSetting(Entity\FarmSetting::TIMEZONE), 'M j, Y');
/* @var $user Entity\Account\User */
if ($n['to'] == 'owner') {
if ($dbFarm->ownerId) {
$user = Entity\Account\User::findPk($dbFarm->ownerId);
} else {
$user = Entity\Account\User::findOne([['accountId' => $dbFarm->ClientID], ['type' => Entity\Account\User::TYPE_ACCOUNT_OWNER]]);
}
if (\Scalr::config('scalr.auth_mode') == 'ldap') {
$email = $user->getSetting(Entity\Account\User\UserSetting::NAME_LDAP_EMAIL);
if (!$email) {
$email = $user->email;
}
} else {
$email = $user->email;
}
$mailer->addTo($email);
} else {
foreach (explode(',', $n['emails']) as $email) {
$mailer->addTo(trim($email));
}
}
$mailer->sendTemplate(SCALR_TEMPLATES_PATH . '/emails/farm_lease_terminate.eml.php', array('terminateDate' => $tdHuman, 'farm' => $dbFarm->Name, 'envName' => $dbFarm->GetEnvironmentObject()->name, 'envId' => $dbFarm->GetEnvironmentObject()->id, 'showOwnerWarning' => !$dbFarm->ownerId));
$notifications[$n['key']] = 1;
$dbFarm->SetSetting(Entity\FarmSetting::LEASE_NOTIFICATION_SEND, json_encode($notifications));
$this->log('INFO', "Notification was sent by key: %s about farm: %s [ID: %d] by lease manager", $n['key'], $dbFarm->Name, $dbFarm->ID);
}
}
}
}
}
} catch (Exception $e) {
throw $e;
}
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$serverId = $request->serverId;
$logger = Logger::getLogger(__CLASS__);
$this->log("INFO", "Processing messages for %s server", $serverId);
try {
$dbserver = DBServer::LoadByID($serverId);
if ($dbserver->farmId) {
if ($dbserver->GetFarmObject()->Status == FARM_STATUS::TERMINATED) {
throw new ServerNotFoundException("Farm related to this server has been terminated.");
}
}
} catch (ServerNotFoundException $e) {
//By some reason server does not exist
$this->db->Execute("\n DELETE m FROM messages m\n WHERE m.server_id = ? AND m.`type` = ? AND m.`status` = ?\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]);
return false;
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
$rs = $this->db->Execute("\n SELECT m.* FROM messages m\n WHERE m.server_id = ? AND m.type = ? AND m.status = ?\n ORDER BY m.dtadded ASC\n ", [$serverId, "in", MESSAGE_STATUS::PENDING]);
while ($row = $rs->FetchRow()) {
try {
if ($row["message_format"] == 'xml') {
$message = $this->serializer->unserialize($row["message"]);
} else {
$message = $this->jsonSerializer->unserialize($row["message"]);
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_MESSAGE_FORMAT, 'json');
}
$message->messageIpAddress = $row['ipaddress'];
$event = null;
$startTime = microtime(true);
// Update scalarizr package version
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]) {
$dbserver->setScalarizrVersion($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]);
}
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]) {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_UPD_CLIENT_VERSION, $message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]);
}
if ($dbserver->GetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES)) {
continue;
}
if ($message instanceof \Scalr_Messaging_Msg) {
$this->log('INFO', "Handling '%s' for '%s' server", $message->getName(), $serverId);
}
try {
if ($message instanceof Scalr_Messaging_Msg_OperationResult) {
if ($message->status == 'ok' || $message->status == 'completed') {
if ($message->name == 'Grow MySQL/Percona data volume' || $message->name == 'mysql.grow-volume') {
$volumeConfig = $message->data ? $message->data : $message->result;
$oldVolumeId = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::VOLUME_ID);
$engine = $dbserver->GetFarmRoleObject()->GetSetting(Scalr_Db_Msr::DATA_STORAGE_ENGINE);
try {
$storageVolume = Scalr_Storage_Volume::init();
try {
$storageVolume->loadById($volumeConfig->id);
$storageVolume->setConfig($volumeConfig);
$storageVolume->save();
} catch (Exception $e) {
if (strpos($e->getMessage(), 'not found')) {
$storageVolume->loadBy(array('id' => $volumeConfig->id, 'client_id' => $dbserver->clientId, 'env_id' => $dbserver->envId, 'name' => "'{$volumeConfig->tags->service}' data volume", 'type' => $engine, 'platform' => $dbserver->platform, 'size' => $volumeConfig->size, 'fstype' => $volumeConfig->fstype, 'purpose' => $volumeConfig->tags->service, 'farm_roleid' => $dbserver->farmRoleId, 'server_index' => $dbserver->index));
$storageVolume->setConfig($volumeConfig);
$storageVolume->save(true);
} else {
throw $e;
}
}
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::VOLUME_ID, $volumeConfig->id, DBFarmRole::TYPE_LCL);
if ($engine == MYSQL_STORAGE_ENGINE::EBS) {
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_EBS_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG);
} elseif ($engine == MYSQL_STORAGE_ENGINE::RAID_EBS) {
$dbserver->GetFarmRoleObject()->SetSetting(Scalr_Db_Msr::DATA_STORAGE_RAID_DISK_SIZE, $volumeConfig->size, DBFarmRole::TYPE_CFG);
}
// Remove old
$storageVolume->delete($oldVolumeId);
} catch (Exception $e) {
Logger::getLogger(__CLASS__)->error(new FarmLogMessage($dbserver->farmId, "Cannot save storage volume: {$e->getMessage()}"));
}
}
} elseif ($message->status == 'error' || $message->status == 'failed') {
if ($message->name == 'Initialization' || $message->name == 'system.init') {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED, 1);
if (is_object($message->error)) {
$errorText = $message->error->message;
} elseif ($message->error) {
$errorText = $message->error;
}
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText);
$event = new HostInitFailedEvent($dbserver, $errorText);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_InitFailed) {
$errorText = $message->reason;
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG, $errorText);
$event = new HostInitFailedEvent($dbserver, $errorText);
} elseif ($message instanceof \Scalr_Messaging_Msg_RuntimeError) {
$logger->fatal(new FarmLogMessage($dbserver->farmId, "Scalarizr failed to launch on server '{$dbserver->getNameByConvention()}' with runtime error: {$message->message}", $dbserver->serverId));
} elseif ($message instanceof Scalr_Messaging_Msg_UpdateControlPorts) {
$apiPort = $message->api;
$ctrlPort = $message->messaging;
$snmpPort = $message->snmp;
// Check API port;
$currentApiPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$currentApiPort) {
$currentApiPort = 8010;
}
if ($apiPort && $apiPort != $currentApiPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr API port was changed from {$currentApiPort} to {$apiPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_API_PORT, $apiPort);
}
// Check Control port
$currentCtrlPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT);
if (!$currentCtrlPort) {
$currentCtrlPort = 8013;
}
if ($ctrlPort && $ctrlPort != $currentCtrlPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr Control port was changed from {$currentCtrlPort} to {$ctrlPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT, $ctrlPort);
}
//Check SNMP port
$currentSnmpPort = $dbserver->GetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT);
if (!$currentSnmpPort) {
$currentSnmpPort = 8014;
}
if ($snmpPort && $snmpPort != $currentSnmpPort) {
$logger->warn(new FarmLogMessage($dbserver->farmId, "Scalarizr SNMP port was changed from {$currentSnmpPort} to {$snmpPort}", $dbserver->serverId));
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, $snmpPort);
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_HostDown) {
$status = PlatformFactory::NewPlatform($dbserver->platform)->GetServerRealStatus($dbserver);
if ($status->isRunning()) {
$event = new RebootBeginEvent($dbserver);
} else {
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
if (!$status->isTerminated()) {
//Stopping
$logger->error(new FarmLogMessage($dbserver->farmId, "Server is in '{$status->getName()}' state. Ignoring HostDown event.", $dbserver->serverId));
$isStopping = true;
}
}
if ($isStopping) {
$dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$dbserver->remoteIp = "";
$dbserver->localIp = "";
$dbserver->status = SERVER_STATUS::SUSPENDED;
$dbserver->Save();
}
$event = new HostDownEvent($dbserver);
$event->isSuspended = true;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_PrepareBundleResult) {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
} catch (Exception $e) {
}
if ($bundleTask) {
if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::PREPARING) {
if ($message->status == 'ok') {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'os' => $message->os, 'software' => $message->software);
$bundleTask->setMetaData($metaData);
$bundleTask->Save();
PlatformFactory::NewPlatform($bundleTask->platform)->CreateServerSnapshot($bundleTask);
} else {
$bundleTask->SnapshotCreationFailed("PrepareBundle procedure failed: {$message->lastError}");
}
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DeployResult) {
try {
$deploymentTask = Scalr_Model::init(Scalr_Model::DM_DEPLOYMENT_TASK)->loadById($message->deployTaskId);
} catch (Exception $e) {
}
if ($deploymentTask) {
if ($message->status == 'error') {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_FAILED;
$deploymentTask->lastError = $message->lastError;
} else {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_DEPLOYED;
$deploymentTask->dtDeployed = date("Y-m-d H:i:s");
}
$deploymentTask->save();
}
} elseif ($message instanceof Scalr_Messaging_Msg_Hello) {
$event = $this->onHello($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_FireEvent) {
//Validate event
$isEventExist = $this->db->GetOne("\n SELECT id FROM event_definitions\n WHERE name = ? AND ((env_id = ? AND account_id = ?) OR (env_id IS NULL AND account_id = ?) OR (env_id IS NULL AND account_id IS NULL))\n LIMIT 1\n ", array($message->eventName, $dbserver->envId, $dbserver->clientId, $dbserver->clientId));
if ($isEventExist) {
$event = new CustomEvent($dbserver, $message->eventName, (array) $message->params);
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostUpdate) {
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_MongoDb) {
/********* MONGODB *********/
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DbMsr) {
/********* DBMSR *********/
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostInit) {
$event = $this->onHostInit($message, $dbserver);
try {
$dbserver->updateTimelog('ts_hi', $message->secondsSinceBoot, $message->secondsSinceStart);
} catch (Exception $e) {
}
if (!$event) {
continue;
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostUp) {
$event = $this->onHostUp($message, $dbserver);
try {
$dbserver->updateTimelog('ts_hu');
} catch (Exception $e) {
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostDown) {
$ignoreHostDown = false;
$p = PlatformFactory::NewPlatform($dbserver->platform);
$status = $p->GetServerRealStatus($dbserver);
if ($dbserver->isOpenstack()) {
if (stristr($status->getName(), 'REBOOT') || stristr($status->getName(), 'HARD_REBOOT')) {
$logger->error(new FarmLogMessage($dbserver->farmId, "Rackspace server is in " . $status->getName() . " state. Ignoring HostDown message.", $dbserver->serverId));
$isRebooting = true;
}
} elseif ($dbserver->platform == \SERVER_PLATFORMS::GCE) {
if ($status->getName() == 'STOPPING') {
// We don't know is this shutdown or stop so let's ignore HostDown
// and wait for status change
$doNotProcessMessage = true;
$ignoreHostDown = true;
} elseif ($status->getName() == 'RUNNING') {
$isRebooting = true;
} elseif ($status->isSuspended() && $dbserver->status != \SERVER_STATUS::PENDING_TERMINATE) {
$isStopping = true;
}
} else {
if ($p->getResumeStrategy() == 'init') {
//TODO: Check is is stopping or shutting-down procedure.
if (!$status->isTerminated()) {
$isStopping = true;
}
}
}
if ($isStopping) {
$dbserver->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$dbserver->remoteIp = "";
$dbserver->localIp = "";
$dbserver->status = SERVER_STATUS::SUSPENDED;
$dbserver->Save();
$event = new HostDownEvent($dbserver);
$event->isSuspended = true;
} elseif ($isRebooting) {
$event = new RebootBeginEvent($dbserver);
} elseif (!$ignoreHostDown) {
if ($dbserver->farmId) {
$wasHostDownFired = $this->db->GetOne("SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($dbserver->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbserver);
}
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_RebootStart) {
$event = new RebootBeginEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_RebootFinish) {
if (!$dbserver->localIp && $message->localIp) {
$dbserver->localIp = $message->localIp;
$dbserver->Save();
}
$event = new RebootCompleteEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_BeforeHostUp) {
$event = new BeforeHostUpEvent($dbserver);
try {
$dbserver->updateTimelog('ts_bhu');
} catch (Exception $e) {
}
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceAttached) {
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION));
$instanceId = $dbserver->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID);
//The main goal of using filters there is to considerably decrease the size of the response.
$volumes = $aws->ec2->volume->describe(null, array(array('name' => VolumeFilterNameType::attachmentInstanceId(), 'value' => (string) $instanceId), array('name' => VolumeFilterNameType::attachmentDevice(), 'value' => (string) $message->deviceName), array('name' => VolumeFilterNameType::status(), 'value' => AMAZON_EBS_STATE::IN_USE)));
foreach ($volumes as $volume) {
/* @var $volume Scalr\Service\Aws\Ec2\DataType\VolumeData */
if ($volume->status == AMAZON_EBS_STATE::IN_USE && count($volume->attachmentSet) && $volume->attachmentSet[0]->instanceId == $instanceId && $volume->attachmentSet[0]->device == $message->deviceName) {
$message->volumeId = $volume->volumeId;
}
}
//Releases memory
unset($volumes);
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
}
$event = new EBSVolumeAttachedEvent($dbserver, $message->deviceName, $message->volumeId);
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceMounted) {
// Single volume
$ebsinfo = $this->db->GetRow("\n SELECT * FROM ec2_ebs WHERE volume_id=? LIMIT 1\n ", array($message->volumeId));
if ($ebsinfo) {
$this->db->Execute("\n UPDATE ec2_ebs\n SET mount_status=?, isfsexist='1'\n WHERE id=?\n ", array(EC2_EBS_MOUNT_STATUS::MOUNTED, $ebsinfo['id']));
}
$event = new EBSVolumeMountedEvent($dbserver, $message->mountpoint, $message->volumeId, $message->deviceName);
} elseif ($message instanceof Scalr_Messaging_Msg_RebundleResult) {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_OK) {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'dist' => $message->dist, 'os' => $message->os, 'software' => $message->software);
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
if ($message->aws) {
if ($message->aws->rootDeviceType == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
}
if ($message->aws->virtualizationType == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
} else {
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver);
try {
$info = $aws->ec2->image->describe($dbserver->GetProperty(EC2_SERVER_PROPERTIES::AMIID))->get(0);
if ($info->rootDeviceType == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
} else {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) {
$tags[] = ROLE_TAGS::EC2_EBS;
}
} catch (Exception $e) {
}
}
if ($info->virtualizationType == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
unset($info);
} catch (Exception $e) {
$metaData['tagsError'] = $e->getMessage();
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) {
$tags[] = ROLE_TAGS::EC2_EBS;
}
} catch (Exception $e) {
}
}
//Releases memory
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
}
}
$metaData['tags'] = $tags;
$event = new RebundleCompleteEvent($dbserver, $message->snapshotId, $message->bundleTaskId, $metaData);
} else {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_FAILED) {
$event = new RebundleFailedEvent($dbserver, $message->bundleTaskId, $message->lastError);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateDataBundleResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE, array('snapshotConfig' => $message->snapshotConfig, 'logFile' => $message->logFile, 'logPos' => $message->logPos, 'dataBundleSize' => $message->dataBundleSize, 'snapshotId' => $message->snapshotId));
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateBackupResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP, array());
$event->backupParts = $message->backupParts;
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_PromoteToMasterResult) {
$event = $this->onMysql_PromoteToMasterResult($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreatePmaUserResult) {
$farmRole = DBFarmRole::LoadByID($message->farmRoleId);
if ($message->status == "ok") {
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_USER, $message->pmaUser, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_PASS, $message->pmaPassword, DBFarmRole::TYPE_LCL);
} else {
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_ERROR, $message->lastError, DBFarmRole::TYPE_LCL);
}
} elseif ($message instanceof Scalr_Messaging_Msg_RabbitMq_SetupControlPanelResult) {
$farmRole = $dbserver->GetFarmRoleObject();
if ($message->status == "ok") {
$mgmtHost = $dbserver->getSzrHost();
if ($message->port) {
$mgmtURL = "http://{$mgmtHost}:{$message->port}/mgmt";
} elseif ($message->cpanelUrl) {
$info = parse_url($message->cpanelUrl);
$mgmtURL = "http://{$mgmtHost}:{$info['port']}/mgmt";
}
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, $dbserver->serverId, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_URL, $mgmtURL, DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
} else {
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "", DBFarmRole::TYPE_LCL);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_ERROR_MSG, $message->lastError, DBFarmRole::TYPE_LCL);
}
} elseif ($message instanceof Scalr_Messaging_Msg_AmiScriptsMigrationResult) {
try {
//Open security group:
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$info = PlatformFactory::NewPlatform($dbserver->platform)->GetServerExtendedInformation($dbserver);
$sg = explode(", ", $info['Security groups']);
foreach ($sg as $sgroup) {
if ($sgroup != 'default') {
// For Scalarizr
$group_rules = array(array('rule' => 'tcp:8013:8013:0.0.0.0/0'), array('rule' => 'udp:8014:8014:0.0.0.0/0'));
$aws = $dbserver->GetEnvironmentObject()->aws($dbserver);
$ipPermissions = new \Scalr\Service\Aws\Ec2\DataType\IpPermissionList();
foreach ($group_rules as $rule) {
$group_rule = explode(":", $rule["rule"]);
$ipPermissions->append(new \Scalr\Service\Aws\Ec2\DataType\IpPermissionData($group_rule[0], $group_rule[1], $group_rule[2], new \Scalr\Service\Aws\Ec2\DataType\IpRangeData($group_rule[3])));
}
$aws->ec2->securityGroup->authorizeIngress($ipPermissions, null, $sgroup);
$dbserver->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
unset($ipPermissions);
break;
}
}
}
} catch (Exception $e) {
$logger->fatal($e->getMessage());
}
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, 8014);
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, "0.7.217");
if ($message->mysql) {
$event = $this->onHostUp($message, $dbserver, true);
}
}
$handle_status = MESSAGE_STATUS::HANDLED;
} catch (Exception $e) {
$handle_status = MESSAGE_STATUS::FAILED;
$logger->error(sprintf("Cannot handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s). %s", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId, $e->getMessage() . "({$e->getFile()}:{$e->getLine()})"));
}
if (!$doNotProcessMessage) {
$totalTime = microtime(true) - $startTime;
$this->db->Execute("\n UPDATE messages\n SET status = ?, processing_time = ?, dtlasthandleattempt = NOW()\n WHERE messageid = ?\n ", array($handle_status, $totalTime, $message->messageId));
} else {
$logger->info(sprintf("Handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s) is postponed due to status transition", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId));
}
if ($event) {
\Scalr::FireEvent($dbserver->farmId, $event);
}
} catch (Exception $e) {
$logger->error($e->getMessage());
}
}
return $request;
}
public function StartThread($bundle_task_info)
{
$db = \Scalr::getDb();
// Reconfigure observers;
Scalr::ReconfigureObservers();
$BundleTask = BundleTask::LoadById($bundle_task_info['id']);
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
} catch (\Scalr\Exception\ServerNotFoundException $e) {
if (!$BundleTask->snapshotId) {
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$BundleTask->setDate('finished');
$BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId);
$BundleTask->Save();
return;
}
} catch (Exception $e) {
//$this->Logger->error($e->getMessage());
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION:
$ctrlPort = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_CTRL_PORT);
if (!$ctrlPort) {
$ctrlPort = 8013;
}
if (\Scalr::config('scalr.instances_connection_policy') == 'local') {
$requestHost = $DBServer->localIp;
} elseif (\Scalr::config('scalr.instances_connection_policy') == 'public') {
$requestHost = $DBServer->remoteIp;
} elseif (\Scalr::config('scalr.instances_connection_policy') == 'auto') {
if ($DBServer->remoteIp) {
$requestHost = $DBServer->remoteIp;
} else {
$requestHost = $DBServer->localIp;
}
}
$conn = @fsockopen($requestHost, $ctrlPort, $errno, $errstr, 10);
if ($conn) {
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1);
$BundleTask->Log("Outbound connection successfully established. Awaiting user action: prebuild automation selection");
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION;
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
$BundleTask->Save();
} else {
$errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $requestHost, $ctrlPort, $errstr);
$errMsg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR);
if (!$errMsg || $errstr != $errMsg) {
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr);
$BundleTask->Log("{$errstr} Will try again in a few minutes.");
}
}
exit;
break;
case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION:
//NOTHING TO DO;
exit;
break;
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$BundleTask->setDate('started');
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($DBServer->platform)->GetServerID($DBServer)) {
$BundleTask->Log(sprintf(_("Waiting for temporary server")));
exit;
}
if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
// IF server is in pensing state
$status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer);
if ($status->isPending()) {
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
$BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName()));
exit;
} elseif ($status->isTerminated()) {
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
break;
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$DBServer->remoteIp = $ips['remoteIp'];
$DBServer->localIp = $ips['localIp'];
$DBServer->save();
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$BundleTask->save();
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$BundleTask->Log(sprintf(_("Initializing SSH2 session to the server")));
if ($DBServer->platform == SERVER_PLATFORMS::IDCF && !$DBServer->remoteIp) {
try {
$BundleTask->Log("Creating port forwarding rules to be able to connect to the server by SSH");
$environment = $DBServer->GetEnvironmentObject();
$cloudLocation = $DBServer->GetCloudLocation();
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$sharedIpId = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SHARED_IP_ID . ".{$cloudLocation}", $environment, false);
$sharedIp = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SHARED_IP . ".{$cloudLocation}", $environment, false);
$BundleTask->Log("Shared IP: {$sharedIp}");
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $environment), $DBServer->platform);
// Create port forwarding rules for scalarizr
$port = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false);
if (!$port) {
$port1 = 30000;
$port2 = 30001;
$port3 = 30002;
$port4 = 30003;
} else {
$port1 = $port + 1;
$port2 = $port1 + 1;
$port3 = $port2 + 1;
$port4 = $port3 + 1;
}
$result2 = $cs->createPortForwardingRule($sharedIpId, 8014, "udp", $port1, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
$result1 = $cs->createPortForwardingRule($sharedIpId, 8013, "tcp", $port1, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
$result3 = $cs->createPortForwardingRule($sharedIpId, 8010, "tcp", $port3, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
$result4 = $cs->createPortForwardingRule($sharedIpId, 8008, "tcp", $port2, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
$result5 = $cs->createPortForwardingRule($sharedIpId, 22, "tcp", $port4, $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID));
$DBServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4));
$DBServer->remoteIp = $sharedIp;
$DBServer->Save();
$platform->setConfigVariable(array(Modules_Platforms_Cloudstack::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false);
} catch (Exception $e) {
$BundleTask->Log("Unable to create port-forwarding rules: {$e->getMessage()}");
}
exit;
}
if ($DBServer->platform == SERVER_PLATFORMS::ECS && !$DBServer->remoteIp) {
$BundleTask->Log(sprintf(_("Server doesn't have public IP. Assigning...")));
$osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION));
$ports = $osClient->network->ports->list();
foreach ($ports as $port) {
if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$serverNetworkPort = $port->id;
break;
}
}
$ips = $osClient->network->floatingIps->list();
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ipId = false;
$ipInfo = false;
foreach ($ips as $ip) {
if ($ip->port_id && $ip->port_id == $serverNetworkPort) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipAssigned = true;
$ipInfo = $ip;
break;
}
if (!$ip->fixed_ip_address && !$ipAddress) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipInfo = $ip;
}
}
if (!$ipAssigned) {
if (!$serverNetworkPort) {
$BundleTask->Log("Unable to identify network port of instance");
exit;
} else {
if (!$ipAddress) {
$networks = $osClient->network->listNetworks();
foreach ($networks as $network) {
if ($network->{"router:external"} == true) {
$publicNetworkId = $network->id;
}
}
if (!$publicNetworkId) {
$BundleTask->Log("Unable to identify public network to allocate");
exit;
} else {
$ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort);
$ipAddress = $ip->floating_ip_address;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
$BundleTask->Log("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}");
}
} else {
$BundleTask->Log("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")");
$osClient->network->floatingIps->update($ipId, $serverNetworkPort);
}
}
} else {
$BundleTask->Log("IP: {$ipAddress} already assigned");
}
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
exit;
}
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
//Prepare script
$BundleTask->Log(sprintf(_("Uploading builder scripts...")));
$behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
if ($DBServer->isOpenstack()) {
$platform = SERVER_PLATFORMS::OPENSTACK;
} else {
$platform = $DBServer->platform;
}
$baseUrl = \Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host');
$options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $DBServer->envId, 'region' => $DBServer->GetCloudLocation(), 'scalr-id' => SCALR_ID);
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
/*
%CHEF_SERVER_URL%
%CHEF_VALIDATOR_NAME%
%CHEF_VALIDATOR_KEY%
%CHEF_ENVIRONMENT%
%CHEF_ROLE_NAME%
*/
$chefServerId = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID);
if ($chefServerId) {
$chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId));
$chefServerInfo['v_auth_key'] = $this->crypto->decrypt($chefServerInfo['v_auth_key'], $this->cryptoKey);
}
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
/*
$BundleTask->Log(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
*/
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
// For CGE we need to use sudo
if ($BundleTask->platform == SERVER_PLATFORMS::GCE || $BundleTask->osFamily == 'amazon') {
$shell = $ssh2Client->getShell();
fwrite($shell, "sudo su\n");
fwrite($shell, "touch /var/log/role-builder-output.log 2>&1\n");
fwrite($shell, "chmod 0666 /var/log/role-builder-output.log 2>&1\n");
fwrite($shell, "setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &\n");
// Get output
$iterations = 0;
while ($l = @fgets($shell, 4096) && $iterations < 10) {
$meta = stream_get_meta_data($shell);
if ($meta["timed_out"]) {
break;
}
$retval .= $l;
$iterations++;
}
@fclose($shell);
$BundleTask->Log("Exec result: {$retval}");
/*
$r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log");
$BundleTask->Log("1: {$r1} ({$ssh2Client->stdErr})");
$r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log");
$BundleTask->Log("2: {$r2} ({$ssh2Client->stdErr})");
$r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$BundleTask->Log("3: {$r3} ({$ssh2Client->stdErr})");
*/
} else {
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$BundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, $DBServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog));
$BundleTask->SnapshotCreationFailed($msg);
} else {
$BundleTask->Log($msg);
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($BundleTask->platform);
$platformModule->CreateServerSnapshot($BundleTask);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage());
$BundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($BundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS:
$r_farm_roles = array();
$BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType));
try {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$BundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=? AND new_role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
$update_farm_dns_zones = array();
$completed_roles = 0;
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) {
$BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
$completed_roles++;
} else {
$servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. " . "Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
if (count($servers) == 0) {
$DBFarmRole->RoleID = $DBFarmRole->NewRoleID;
$DBFarmRole->NewRoleID = null;
$DBFarmRole->Save();
$update_farm_dns_zones[$DBFarmRole->FarmID] = 1;
$completed_roles++;
} else {
$metaData = $BundleTask->getSnapshotDetails();
foreach ($servers as $server) {
try {
$DBServer = DBServer::LoadByID($server['server_id']);
} catch (Exception $e) {
//TODO:
continue;
}
if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) {
$DBServer->roleId = $BundleTask->roleId;
$DBServer->Save();
if ($metaData['noServersReplace']) {
$BundleTask->Log(sprintf("'Do not replace servers' option was checked. " . "Server '%s' won't be replaced to new image.", $DBServer->serverId));
} else {
$BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, " . "already has all modifications. No need to replace it.", $DBServer->serverId));
}
if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) {
$DBServer->terminate(array('BUNDLE_TASK_FINISHED', 'Synchronizing', $BundleTask->id));
}
} else {
if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?,?) LIMIT 1", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TROUBLESHOOTING))) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID);
$nDBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Server replacement after snapshotting", !empty($BundleTask->createdById) ? $BundleTask->createdById : null);
$nDBServer->replaceServerID = $DBServer->serverId;
$nDBServer->Save();
$BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId));
}
}
}
}
}
}
if ($completed_roles == count($r_farm_roles)) {
$BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
$DBServer->Remove();
} elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) {
$BundleTask->Log("Terminating temporary server");
$DBServer->terminate('TEMPORARY_SERVER_ROLE_BUILDER');
$BundleTask->Log("Termination request has been sent");
}
} catch (Exception $e) {
$BundleTask->Log("Warning: {$e->getMessage()}");
}
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Save();
}
try {
if (count($update_farm_dns_zones) != 0) {
foreach ($update_farm_dns_zones as $farm_id => $v) {
$dnsZones = DBDNSZone::loadByFarmId($farm_id);
foreach ($dnsZones as $dnsZone) {
if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) {
$dnsZone->updateSystemRecords();
$dnsZone->save();
}
}
}
}
} catch (Exception $e) {
$this->Logger->fatal("DNS ZONE: {$e->getMessage()}");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = DBRole::loadById($DBServer->roleId);
if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($DBServer) {
$new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer);
} else {
$new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
} else {
//TODO:
//$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})");
}
}
try {
$DBRole = DBRole::createFromBundleTask($BundleTask);
} catch (Exception $e) {
$BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
if ($DBServer->farmId) {
// Create DBFarmRole object
// TODO: create DBFarm role
}
}
} catch (Exception $e) {
}
} else {
try {
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS));
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
$DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
} else {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
$DBFarmRole = DBFarmRole::LoadByID($farm_role['id']);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
}
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS;
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
$BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
if ($BundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) {
try {
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
$DBServer->Remove();
} elseif ($DBServer->status == SERVER_STATUS::TEMPORARY) {
$BundleTask->Log("Terminating temporary server");
$DBServer->terminate('TEMPORARY_SERVER_ROLE_BUILDER');
$BundleTask->Log("Termination request has been sent");
}
} catch (Exception $e) {
$BundleTask->Log("Warning: {$e->getMessage()}");
}
}
$BundleTask->Save();
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
}
break;
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm
return;
}
$this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID);
$config = \Scalr::getContainer()->config;
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/* @var $DBServer \DBServer */
try {
if ($DBServer->cloudLocation) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId);
$p = PlatformFactory::NewPlatform($DBServer->platform);
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId));
continue;
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag') {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
//More debug
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache)));
$this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}"));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
/* @var $env \Scalr_Environment */
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
//Saving the reason why this environment is disabled
$env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]);
return;
} elseif (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
$this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
if ($openstackErrorState) {
try {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId));
} catch (Exception $e) {
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer);
$DBServer->Save();
}
}
if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) {
$DBServer->status = \SERVER_STATUS::PENDING;
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1);
$DBServer->dateAdded = date("Y-m-d H:i:s");
$DBServer->Save();
} else {
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
\Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
}
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) {
$DBServer->remoteIp = $ipaddresses['remoteIp'];
$DBServer->localIp = $ipaddresses['localIp'];
$DBServer->Save();
}
//Add tags
Ec2Helper::createServerTags($DBServer);
}
if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scripting_event, $DBServer->farmRoleId]);
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
} elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) {
\Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
return $request;
}
public function StartThread($mysql_farm_role)
{
// Reconfigure observers;
Scalr::ReconfigureObservers();
$db = \Scalr::getDb();
$DBFarmRole = DBFarmRole::LoadByID($mysql_farm_role['id']);
try {
$DBFarm = $DBFarmRole->GetFarmObject();
} catch (Exception $e) {
return;
}
//skip terminated farms
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
return;
}
$tz = Scalr_Environment::init()->loadById($DBFarm->EnvID)->getPlatformConfigValue(Scalr_Environment::SETTING_TIMEZONE);
if ($tz) {
date_default_timezone_set($tz);
}
//
// Check replication status
//
$this->Logger->info("[FarmID: {$DBFarm->ID}] Checking replication status");
$servers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::RUNNING));
//
// Check backups and mysql bandle procedures
//
//Backups
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_ENABLED) && $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) != 0) {
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING) == 1) {
// Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots)
// We must wait for running bundle process.
$bcp_timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) * 60 * 5;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BCP_TS) + $bcp_timeout < time()) {
$bcp_timeouted = true;
}
if (!empty($bcp_timeouted)) {
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING, 0, Entity\FarmRoleSetting::TYPE_LCL);
$this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Backup already running. Timeout. Clear lock.");
}
} else {
$timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BCP_EVERY) * 60;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BCP_TS) + $timeout < time()) {
$this->Logger->info("[FarmID: {$DBFarm->ID}] Need new backup");
$servers = $DBFarm->GetMySQLInstances(false, true);
if (empty($servers[0])) {
$servers = $DBFarm->GetMySQLInstances(true);
} else {
$servers = array_reverse($servers);
}
$DBServer = isset($servers[0]) ? $servers[0] : null;
if ($DBServer) {
if ($DBServer->status == SERVER_STATUS::RUNNING) {
$msg = new Scalr_Messaging_Msg_Mysql_CreateBackup($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_ROOT_PASSWORD));
$DBServer->SendMessage($msg);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BCP_RUNNING, 1, Entity\FarmRoleSetting::TYPE_LCL);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_BCP_SERVER_ID, $DBServer->serverId, Entity\FarmRoleSetting::TYPE_LCL);
}
} else {
$this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql instances for run backup procedure!");
}
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_ENABLED) && $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY) != 0) {
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING) == 1) {
// Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots)
// We must wait for running bundle process.
$bundle_timeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY) * (3600 * 2);
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BUNDLE_TS) + $bundle_timeout < time()) {
$bundle_timeouted = true;
}
if (!empty($bundle_timeouted)) {
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING, 0, Entity\FarmRoleSetting::TYPE_LCL);
$this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Bundle already running. Timeout. Clear lock.");
}
} else {
/*
* Check bundle window
*/
$bundleEvery = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_EVERY);
$timeout = $bundleEvery * 3600;
$lastBundleTime = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_LAST_BUNDLE_TS);
$performBundle = false;
if ($bundleEvery % 24 == 0) {
if ($lastBundleTime) {
$days = $bundleEvery / 24;
$bundleDay = (int) date("md", strtotime("+{$days} day", $lastBundleTime));
if ($bundleDay > (int) date("md")) {
return;
}
}
$pbwFrom = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_MM));
$pbwTo = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_MM));
if ($pbwFrom && $pbwTo) {
$current_time = (int) date("Hi");
if ($pbwFrom <= $current_time && $pbwTo >= $current_time) {
$performBundle = true;
}
} else {
$performBundle = true;
}
} else {
//Check timeout
if ($lastBundleTime + $timeout < time()) {
$pbwFrom = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_START_MM));
$pbwTo = (int) ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_WINDOW_END_MM));
if ($pbwFrom && $pbwTo) {
$current_time = (int) date("Hi");
if ($pbwFrom <= $current_time && $pbwTo >= $current_time) {
$performBundle = true;
}
} else {
$performBundle = true;
}
}
}
if ($performBundle) {
$this->Logger->info("[FarmID: {$DBFarm->ID}] Need mySQL bundle procedure");
// Rebundle
$servers = $DBFarm->GetMySQLInstances(true, false);
$DBServer = isset($servers[0]) ? $servers[0] : null;
if ($DBServer) {
if ($DBServer->status == SERVER_STATUS::RUNNING) {
$DBServer->SendMessage(new Scalr_Messaging_Msg_Mysql_CreateDataBundle());
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_IS_BUNDLE_RUNNING, 1, Entity\FarmRoleSetting::TYPE_LCL);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::MYSQL_BUNDLE_SERVER_ID, $DBServer->serverId, Entity\FarmRoleSetting::TYPE_LCL);
}
} else {
$this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql master instances for run bundle procedure!");
}
}
}
}
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//The list of the suspension information about cloud platforms
$this->aSuspensionInfo = [];
//Speed up poller
if ($this->config()->daemon) {
//Warming up static DI cache
\Scalr::getContainer()->warmup();
}
// Reconfigure observers
\Scalr::ReconfigureObservers();
$DBFarm = DBFarm::LoadByID($request->farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$transactionId = abs(crc32(posix_getpid() . $request->farmId));
$this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform);
$jobStartTime = microtime(true);
//Retrieves the number of either terminated or suspended servers for the farm
$servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers\n WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]);
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
//There are no servers for this farm and platform
return;
}
$this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform);
$config = \Scalr::getContainer()->config;
/*
if ($request->platform) {
$p = PlatformFactory::NewPlatform($request->platform);
$p->ClearCache();
}
*/
$p = PlatformFactory::NewPlatform($request->platform);
foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) {
/* @var $DBServer \DBServer */
//Get platform suspension info
$suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId);
//If the cloud platform is suspended we should not process it
if ($suspensionInfo->isSuspended()) {
continue;
}
try {
//1. We need to check that server is exists in cloud and not missed.
// (On Openstack server can be missed and should not be terminated)
$cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation);
if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) {
try {
$this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform);
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
//For Azure we need to pass resource group instead of cloudLocation
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP));
} else {
$p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
}
//We successfully polled cloud so can resume suspension status for the cloud platform
if ($suspensionInfo->isPendingSuspend()) {
$suspensionInfo->resume();
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage());
continue;
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
$this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage());
continue;
}
if (!$serverInfo) {
if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) {
continue;
} elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId));
continue;
}
$action = 'terminate';
if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) {
$action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server");
}
if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]);
} else {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
}
continue;
}
} else {
//http.persistent.handles.limit must be set to 0 for pecl-http version 1
$this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId);
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]);
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
$this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine());
continue;
}
try {
if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
// If openstack server is in ERROR state we need more details
if ($openstackErrorState) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$status = empty($info['Status']) ? false : $info['Status'];
} catch (Exception $e) {
}
}
if (empty($status)) {
$status = $DBServer->GetRealStatus()->getName();
}
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
//In case the server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId));
$event = new HostDownEvent($DBServer);
$event->isSuspended = true;
\Scalr::FireEvent($DBFarm->ID, $event);
continue;
} else {
if ($DBServer->status != \SERVER_STATUS::RESUMING) {
//If the server was suspended during initialization
//we do not support this and need to terminate this instance
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
} else {
// Need to clear cache, because this situation happens only when cache is stale.
$p->ClearCache();
}
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
if ($DBServer->platform == \SERVER_PLATFORMS::GCE) {
if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') {
continue;
}
}
$update = [];
// For Openstack we need to re-accociate IPs
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) {
if (!$DBServer->remoteIp) {
$update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer);
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::EC2) {
try {
$info = $p->GetServerExtendedInformation($DBServer);
$realStatus = !empty($info['Instance state']) ? $info['Instance state'] : '';
} catch (\Exception $e) {
// no need to do anything here;
}
$this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus);
}
$update['status'] = \SERVER_STATUS::RESUMING;
$update['dateAdded'] = date("Y-m-d H:i:s");
$DBServer->update($update);
unset($update);
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) {
$elasticIpAssigned = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
if (!$DBServer->remoteIp && !$DBServer->localIp) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
$elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer);
if ($elasticIpAddress) {
$ipaddresses['remoteIp'] = $elasticIpAddress;
$DBServer->remoteIp = $elasticIpAddress;
$elasticIpAssigned = true;
}
if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) {
$DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]);
}
//Add tags
Ec2Helper::createObjectTags($DBServer);
}
}
}
if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) {
if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
// Check scalarizr deployment status
$env = $DBServer->GetEnvironmentObject();
$azure = $env->azure();
$info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME));
$extensions = !empty($info->extensions) ? $info->extensions : [];
foreach ($extensions as $extension) {
/* @var $extension ExtensionData */
if ($extension->name == 'scalarizr') {
$extStatus = $extension->statuses[0];
/* @var $extStatus StatusData */
if ($extStatus->level == 'Error') {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]);
}
}
}
}
} else {
AzureHelper::setupScalrAgent($DBServer);
}
}
try {
if ($DBServer->isOpenstack()) {
OpenstackHelper::setServerFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
\Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scriptingEvent = false;
$eventName = null;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$eventName = "hostInit";
$scriptingEvent = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$eventName = "hostUp";
$scriptingEvent = EVENT_TYPE::HOST_UP;
}
if ($scriptingEvent && $dtadded) {
$hasPendingMessages = !!$db->GetOne("\n SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n ", [$DBServer->serverId]);
$scriptingTimeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scriptingEvent, $DBServer->farmRoleId]);
if ($scriptingTimeout) {
$launchTimeout = $launchTimeout + $scriptingTimeout;
}
if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false);
} catch (Exception $err) {
$this->getLogger()->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
//Whether IP address is changed
if (!$DBServer->IsRebooting() && !$elasticIpAssigned) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId));
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO Check health
}
}
} elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) {
//TODO: Terminated outside scalr while in SUSPENDED state
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = $p->GetServerIPAddresses($DBServer);
// Private IP cannot be removed (only changed).
if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
\Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null;
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform);
$db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]);
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() > $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO Check reboot timeout
}
}
} catch (Exception $e) {
if (CloudPlatformSuspensionInfo::isSuspensionException($e)) {
$suspensionInfo->registerError($e->getMessage());
}
if (stristr($e->getMessage(), "not found")) {
$this->getLogger()->fatal($e->getMessage());
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(5);
$this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage());
} else {
$this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage());
}
}
}
$this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime);
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$dtNow = new DateTime('now');
try {
$dbServer = DBServer::LoadByID($request->serverId);
} catch (ServerNotFoundException $e) {
$this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage());
return false;
}
if (!in_array($dbServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
return false;
}
//Skip Locked instances
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) {
return false;
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (in_array($dbServer->status, array(SERVER_STATUS::TERMINATED)) || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) {
try {
$p = PlatformFactory::NewPlatform($dbServer->platform);
$environment = $dbServer->GetEnvironmentObject();
if (!$environment->isPlatformEnabled($dbServer->platform)) {
throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id));
}
if ($dbServer->GetCloudServerID()) {
$serverHistory = $dbServer->getServerHistory();
$isTermination = in_array($dbServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$isSuspension = in_array($dbServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND));
$status = $dbServer->GetRealStatus();
if ($dbServer->isCloudstack()) {
//Workaround for when expunge flag not working and servers stuck in Destroyed state.
$isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed';
} else {
$isTerminated = $status->isTerminated();
}
if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) {
try {
if ($dbServer->farmId != 0) {
try {
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]]));
if ($dbServer->index == 1 && $serversCount > 1) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. " . "Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
return false;
}
}
} catch (Exception $e) {
}
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
}
} catch (Exception $e) {
$this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage());
}
$terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME);
if (!$terminationTime || time() - $terminationTime > 180) {
if ($isTermination) {
$p->TerminateServer($dbServer);
} else {
$p->SuspendServer($dbServer);
}
$dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time());
if ($dbServer->farmId) {
$wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbServer);
$event->isSuspended = !$isTermination;
\Scalr::FireEvent($dbServer->farmId, $event);
}
}
}
} else {
if ($dbServer->status == SERVER_STATUS::TERMINATED) {
if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) {
$errorResolution = true;
$serverHistory->setTerminated();
$dbServer->Remove();
}
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) {
$dbServer->status = SERVER_STATUS::TERMINATED;
$dbServer->Save();
$errorResolution = true;
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) {
$dbServer->status = SERVER_STATUS::SUSPENDED;
$dbServer->remoteIp = '';
$dbServer->localIp = '';
$dbServer->Save();
$errorResolution = true;
}
}
}
if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) {
//Automatic error resolution
$ste->delete();
}
}
} else {
//If there is no cloudserverID we don't need to add this server into server history.
//$serverHistory->setTerminated();
$dbServer->Remove();
}
} catch (InstanceNotFound $e) {
if ($serverHistory) {
$serverHistory->setTerminated();
}
$dbServer->Remove();
} catch (Exception $e) {
if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || stripos($e->getMessage(), "The request you have made requires authentication.") !== false || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) {
//Postpones unsuccessful task for 30 minutes.
$ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage());
$minutes = rand(30, 40);
$ste->retryAfter = new \DateTime('+' . $minutes . ' minutes');
if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
//We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts
$dbServer->Remove();
}
$ste->save();
} else {
$this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage());
throw $e;
}
}
}
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$db = \Scalr::getDb();
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
$bundleTask = BundleTask::LoadById($request->bundleTaskId);
if (!$bundleTask instanceof BundleTask) {
$this->getLogger()->fatal("Could not load bundle task id: %s", $request->bundleTaskId);
return false;
} else {
$this->bundleTask = $bundleTask;
$this->getLogger()->info("Processing bundle task id: %d status: %s serverid: %s", $bundleTask->id, $bundleTask->status, $bundleTask->serverId);
}
try {
$dbServer = DBServer::LoadByID($bundleTask->serverId);
} catch (\Scalr\Exception\ServerNotFoundException $e) {
if (!$bundleTask->snapshotId) {
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$bundleTask->setDate('finished');
$bundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $bundleTask->serverId);
$bundleTask->Save();
return;
}
$this->getLogger()->warn("Could not load server: %s. %s says: %s", $bundleTask->serverId, get_class($e), $e->getMessage());
} catch (Exception $e) {
$this->getLogger()->warn("Could not load server: %s. %s says: %s", $bundleTask->serverId, get_class($e), $e->getMessage());
}
switch ($bundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::ESTABLISHING_COMMUNICATION:
$conn = @fsockopen($dbServer->getSzrHost(), $dbServer->getPort(DBServer::PORT_CTRL), $errno, $errstr, 10);
if ($conn) {
$dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION, 1);
$this->bundleTaskLog("Outbound connection successfully established. Awaiting user action: prebuild automation selection");
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION;
$this->bundleTaskLog(sprintf(_("Bundle task status: %s"), $bundleTask->status));
$bundleTask->Save();
} else {
$errstr = sprintf("Unable to establish outbound (Scalr -> Scalarizr) communication (%s:%s): %s.", $dbServer->getSzrHost(), $dbServer->getPort(DBServer::PORT_CTRL), $errstr);
$errMsg = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR);
if (!$errMsg || $errstr != $errMsg) {
$dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_OUT_CONNECTION_ERROR, $errstr);
$this->bundleTaskLog("{$errstr} Will try again in a few minutes.");
}
}
return false;
case SERVER_SNAPSHOT_CREATION_STATUS::AWAITING_USER_ACTION:
//nothing to do;
return false;
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$bundleTask->setDate('started');
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($dbServer->platform)->GetServerID($dbServer)) {
$this->bundleTaskLog(sprintf(_("Waiting for temporary server")));
return false;
}
$status = PlatformFactory::NewPlatform($dbServer->platform)->GetServerRealStatus($dbServer);
if ($status->isPending()) {
//Server is in pensing state
$this->bundleTaskLog(sprintf(_("Server status: %s"), $status->getName()));
$this->bundleTaskLog(sprintf(_("Waiting for running state."), $status->getName()));
return false;
} elseif ($status->isTerminated()) {
$this->bundleTaskLog(sprintf(_("Server status: %s"), $status->getName()));
$dbServer->status = SERVER_STATUS::TERMINATED;
$dbServer->save();
$bundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
return false;
}
break;
}
$this->getLogger()->info("Continue bundle task id:%d status:%s", $bundleTask->id, $bundleTask->status);
switch ($bundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($dbServer->platform)->GetServerIPAddresses($dbServer);
$dbServer->remoteIp = $ips['remoteIp'];
$dbServer->localIp = $ips['localIp'];
$dbServer->save();
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$bundleTask->save();
$this->bundleTaskLog(sprintf(_("Bundle task status: %s"), $bundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$this->bundleTaskLog(sprintf(_("Initializing SSH2 session to the server")));
if ($dbServer->platform == SERVER_PLATFORMS::IDCF && !$dbServer->remoteIp) {
try {
$this->bundleTaskLog("Creating port forwarding rules to be able to connect to the server by SSH");
$environment = $dbServer->GetEnvironmentObject();
$cloudLocation = $dbServer->GetCloudLocation();
$platform = PlatformFactory::NewPlatform($dbServer->platform);
$sharedIpId = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP_ID . ".{$cloudLocation}", $environment, false);
$sharedIp = $platform->getConfigVariable(CloudstackPlatformModule::SHARED_IP . ".{$cloudLocation}", $environment, false);
$this->bundleTaskLog("Shared IP: {$sharedIp}");
$cs = $environment->cloudstack($dbServer->platform);
// Create port forwarding rules for scalarizr
$port = $platform->getConfigVariable(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}", $environment, false);
if (!$port) {
$port1 = 30000;
$port2 = 30001;
$port3 = 30002;
$port4 = 30003;
} else {
$port1 = $port + 1;
$port2 = $port1 + 1;
$port3 = $port2 + 1;
$port4 = $port3 + 1;
}
$virtualmachineid = $dbServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID);
$result2 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8014, 'protocol' => "udp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid));
$result1 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8013, 'protocol' => "tcp", 'publicport' => $port1, 'virtualmachineid' => $virtualmachineid));
$result3 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8010, 'protocol' => "tcp", 'publicport' => $port3, 'virtualmachineid' => $virtualmachineid));
$result4 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 8008, 'protocol' => "tcp", 'publicport' => $port2, 'virtualmachineid' => $virtualmachineid));
$result5 = $cs->firewall->createPortForwardingRule(array('ipaddressid' => $sharedIpId, 'privateport' => 22, 'protocol' => "tcp", 'publicport' => $port4, 'virtualmachineid' => $virtualmachineid));
$dbServer->SetProperties(array(SERVER_PROPERTIES::SZR_CTRL_PORT => $port1, SERVER_PROPERTIES::SZR_SNMP_PORT => $port1, SERVER_PROPERTIES::SZR_API_PORT => $port3, SERVER_PROPERTIES::SZR_UPDC_PORT => $port2, SERVER_PROPERTIES::CUSTOM_SSH_PORT => $port4));
$dbServer->remoteIp = $sharedIp;
$dbServer->Save();
$platform->setConfigVariable(array(CloudstackPlatformModule::SZR_PORT_COUNTER . ".{$cloudLocation}.{$sharedIpId}" => $port4), $environment, false);
} catch (Exception $e) {
$this->bundleTaskLog("Unable to create port-forwarding rules: {$e->getMessage()}");
}
return false;
}
if ($dbServer->platform == SERVER_PLATFORMS::ECS && !$dbServer->remoteIp) {
$this->bundleTaskLog(sprintf(_("Server doesn't have public IP. Assigning...")));
$osClient = $dbServer->GetEnvironmentObject()->openstack($dbServer->platform, $dbServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION));
$ports = $osClient->network->ports->list();
foreach ($ports as $port) {
if ($port->device_id == $dbServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$serverNetworkPort = $port->id;
break;
}
}
$ips = $osClient->network->floatingIps->list();
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ipId = false;
$ipInfo = false;
foreach ($ips as $ip) {
if ($ip->port_id && $ip->port_id == $serverNetworkPort) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipAssigned = true;
$ipInfo = $ip;
break;
}
if (!$ip->fixed_ip_address && !$ipAddress) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipInfo = $ip;
}
}
if (!$ipAssigned) {
if (!$serverNetworkPort) {
$this->bundleTaskLog("Unable to identify network port of instance");
return false;
} else {
if (!$ipAddress) {
$networks = $osClient->network->listNetworks();
foreach ($networks as $network) {
if ($network->{"router:external"} == true) {
$publicNetworkId = $network->id;
}
}
if (!$publicNetworkId) {
$this->bundleTaskLog("Unable to identify public network to allocate");
return false;
} else {
$ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort);
$ipAddress = $ip->floating_ip_address;
$dbServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
$this->bundleTaskLog("Allocated new IP {$ipAddress} for port: {$serverNetworkPort}");
}
} else {
$this->bundleTaskLog("Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")");
$osClient->network->floatingIps->update($ipId, $serverNetworkPort);
}
}
} else {
$this->bundleTaskLog("IP: {$ipAddress} already assigned");
}
if ($ipAddress) {
$dbServer->remoteIp = $ipAddress;
$dbServer->Save();
}
return false;
}
try {
$ssh2Client = $dbServer->GetSsh2Client();
$ssh2Client->connect($dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$this->bundleTaskLog(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
return false;
}
$this->bundleTaskLog(sprintf(_("Created SSH session. Username: %s"), $ssh2Client->getLogin()));
//Prepare script
$this->bundleTaskLog(sprintf(_("Uploading builder scripts...")));
$behaviors = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
if ($dbServer->isOpenstack()) {
$platform = SERVER_PLATFORMS::OPENSTACK;
} else {
$platform = $dbServer->platform;
}
$baseUrl = rtrim(\Scalr::config('scalr.endpoint.scheme') . "://" . \Scalr::config('scalr.endpoint.host'), '/');
$options = array('server-id' => $dbServer->serverId, 'role-name' => $bundleTask->roleName, 'crypto-key' => $dbServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $platform, 'queryenv-url' => $baseUrl . "/query-env", 'messaging-p2p.producer-url' => $baseUrl . "/messaging", 'behaviour' => trim(trim(str_replace("base", "", $behaviors), ",")), 'env-id' => $dbServer->envId, 'region' => $dbServer->GetCloudLocation(), 'scalr-id' => SCALR_ID);
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
/*
%CHEF_SERVER_URL%
%CHEF_VALIDATOR_NAME%
%CHEF_VALIDATOR_KEY%
%CHEF_ENVIRONMENT%
%CHEF_ROLE_NAME%
*/
$chefServerId = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_SERVER_ID);
if ($chefServerId) {
$chefServerInfo = $db->GetRow("SELECT * FROM services_chef_servers WHERE id=?", array($chefServerId));
$chefServerInfo['v_auth_key'] = \Scalr::getContainer()->crypto->decrypt($chefServerInfo['v_auth_key']);
}
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_ROLE_NAME%", "%CHEF_NODE_NAME%", "\r\n"), array($platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', $chefServerInfo['url'], $chefServerInfo['v_username'], $chefServerInfo['v_auth_key'], $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ENVIRONMENT), $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_CHEF_ROLE_NAME), '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
/*
$this->bundleTaskLog(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
*/
} catch (Exception $e) {
$this->bundleTaskLog(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
return false;
}
$this->bundleTaskLog("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
// For CGE we need to use sudo
if ($bundleTask->platform == SERVER_PLATFORMS::GCE || $bundleTask->osFamily == 'amazon') {
$shell = $ssh2Client->getShell();
@stream_set_blocking($shell, true);
@stream_set_timeout($shell, 5);
@fwrite($shell, "sudo touch /var/log/role-builder-output.log 2>&1" . PHP_EOL);
$output = @fgets($shell, 4096);
$this->bundleTaskLog("Verbose 1: {$output}");
@fwrite($shell, "sudo chmod 0666 /var/log/role-builder-output.log 2>&1" . PHP_EOL);
$output2 = @fgets($shell, 4096);
$this->bundleTaskLog("Verbose 2: {$output2}");
@fwrite($shell, "sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &" . PHP_EOL);
$output3 = @fgets($shell, 4096);
$this->bundleTaskLog("Verbose 3: {$output3}");
sleep(5);
$meta = stream_get_meta_data($shell);
$this->bundleTaskLog(sprintf("Verbose (Meta): %s", json_encode($meta)));
$i = 4;
if ($meta['eof'] == false && $meta['unread_bytes'] != 0) {
$output4 = @fread($shell, $meta['unread_bytes']);
$this->bundleTaskLog("Verbose {$i}: {$output4}");
$meta = stream_get_meta_data($shell);
$this->bundleTaskLog(sprintf("Verbose (Meta): %s", json_encode($meta)));
}
@fclose($shell);
/*
$r1 = $ssh2Client->exec("sudo touch /var/log/role-builder-output.log");
$this->bundleTaskLog("1: {$r1} ({$ssh2Client->stdErr})");
$r2 = $ssh2Client->exec("sudo chmod 0666 /var/log/role-builder-output.log");
$this->bundleTaskLog("2: {$r2} ({$ssh2Client->stdErr})");
$r3 = $ssh2Client->exec("sudo setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$this->bundleTaskLog("3: {$r3} ({$ssh2Client->stdErr})");
*/
} else {
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
}
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$bundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $dbServer->GetSsh2Client();
$ssh2Client->connect($dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH));
} catch (Exception $e) {
$this->bundleTaskLog(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $dbServer->remoteIp, $dbServer->getPort(DBServer::PORT_SSH), $e->getMessage()));
//TODO: Set status of bundle log to failed
return false;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $dbServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$this->bundleTaskLog(sprintf("role-builder-step.log: %s", $stepLog));
$bundleTask->SnapshotCreationFailed($msg);
} else {
$this->bundleTaskLog($msg);
$dbServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($bundleTask->platform);
$platformModule->CreateServerSnapshot($bundleTask);
} catch (Exception $e) {
$this->getLogger()->error($e->getMessage());
$bundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($bundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$bundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($bundleTask->platform)->CheckServerSnapshotStatus($bundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($bundleTask->object == BundleTask::BUNDLETASK_OBJECT_IMAGE) {
if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$dbRole = $dbServer->GetFarmRoleObject()->GetRoleObject();
$dbRole->__getNewRoleObject()->setImage($bundleTask->platform, $bundleTask->cloudLocation, $bundleTask->snapshotId, $bundleTask->createdById, $bundleTask->createdByEmail);
$this->bundleTaskLog(sprintf(_("Image replacement completed.")));
}
$this->bundleTaskLog(sprintf(_("Bundle task completed.")));
$bundleTask->setDate('finished');
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$bundleTask->Save();
} elseif ($bundleTask->object == BundleTask::BUNDLETASK_OBJECT_ROLE) {
if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = $dbServer->GetFarmRoleObject()->GetRoleObject();
if ($dbRole->name == $bundleTask->roleName && $dbRole->envId == $bundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($dbServer) {
$new_role_name = BundleTask::GenerateRoleName($dbServer->GetFarmRoleObject(), $dbServer);
} else {
$new_role_name = $bundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$this->bundleTaskLog(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $bundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
}
}
try {
$DBRole = DBRole::createFromBundleTask($bundleTask);
} catch (Exception $e) {
$bundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$bundleTask->setDate('finished');
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$this->bundleTaskLog(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
} else {
try {
$this->bundleTaskLog(sprintf(_("Replacement type: %s"), $bundleTask->replaceType));
$r_farm_roles = array();
try {
$DBFarm = DBFarm::LoadByID($bundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$bundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($bundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($bundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("\n SELECT id FROM farm_roles\n WHERE role_id=?\n AND farmid IN (SELECT id FROM farms WHERE env_id=?)\n ", array($bundleTask->prototypeRoleId, $bundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $bundleTask->cloudLocation) {
$this->bundleTaskLog(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role " . "but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
} else {
$DBFarmRole->RoleID = $bundleTask->roleId;
$DBFarmRole->Save();
}
}
$this->bundleTaskLog(sprintf(_("Replacement completed. Bundle task completed.")));
try {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$dbServer->Remove();
} elseif ($dbServer->status == SERVER_STATUS::TEMPORARY) {
$this->bundleTaskLog("Terminating temporary server");
$dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
$this->bundleTaskLog("Termination request has been sent");
}
} catch (Exception $e) {
$this->bundleTaskLog("Warning: {$e->getMessage()}");
}
$bundleTask->setDate('finished');
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$bundleTask->Save();
} catch (Exception $e) {
$this->getLogger()->error($e->getMessage());
$this->bundleTaskLog(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$bundleTask->setDate('finished');
$bundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
}
if ($bundleTask->status == SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS) {
try {
if ($dbServer->status == SERVER_STATUS::IMPORTING) {
$dbServer->Remove();
} elseif ($dbServer->status == SERVER_STATUS::TEMPORARY) {
$this->bundleTaskLog("Terminating temporary server");
$dbServer->terminate(DBServer::TERMINATE_REASON_TEMPORARY_SERVER_ROLE_BUILDER);
$this->bundleTaskLog("Termination request has been sent");
}
} catch (Exception $e) {
$this->bundleTaskLog("Warning: {$e->getMessage()}");
}
}
$bundleTask->Save();
} catch (Exception $e) {
$this->getLogger()->error($e->getMessage());
}
break;
}
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (!isset($request->farmRoleId)) {
//This is the farm with synchronous launch of roles
try {
$DBFarm = DBFarm::LoadByID($request->farmId);
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID);
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId);
throw $e;
}
//Gets the list of the roles
$list = $DBFarm->GetFarmRoles();
} else {
//This is asynchronous lauhch
try {
$DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId);
if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) {
//We don't need to handle inactive farms
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId);
throw $e;
}
$list = [$DBFarmRole];
}
$this->getLogger()->debug("Processing %s FarmRoles", count($list));
foreach ($list as $DBFarmRole) {
// Set Last polling time
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$disabledScaling = false;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') {
if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
// For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running)
// When 1 instance is already running, the rest is fully manual
$roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount();
if ($roleTotalInstances != 0) {
$disabledScaling = true;
}
} else {
if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$disabledScaling = true;
}
}
if ($disabledScaling) {
$this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias);
continue;
}
}
$farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name;
// Get current count of running and pending instances.
$this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
//Replacing the logger
$scalingManager->logger = $this->getLogger();
$scalingDecision = $scalingManager->makeScalingDecision();
$scalingDecisionDetails = $scalingManager->decisonInfo;
$this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails));
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue;
} else {
if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instances count after scaling
resolution the spare instances are running for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null));
continue;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
$ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR);
$useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN);
$isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ);
// Select instance that will be terminated
//
// Instances ordered by uptime (oldest wil be choosen)
// Instance cannot be mysql master
// Choose the one that was rebundled recently
$DBServer = null;
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($isRabbitMQ) {
$serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]);
if ($DBServer->index == 1 && $serverExists) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
$got_valid_instance = true;
}
//Check safe shutdown
if ($useSafeShutdown == 1) {
try {
$res = $DBServer->scalarizr->system->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
if ($res != '1') {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
$got_valid_instance = false;
}
}
}
// end while
if ($DBServer !== null && $got_valid_instance) {
$this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$ignoreFullHour) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
try {
if ($terminateStrategy == 'terminate') {
$DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} else {
$DBServer->suspend('SCALING_DOWN', false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
} catch (Exception $e) {
$this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId));
}
}
} else {
$this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID));
}
//break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution 'need more instances' for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias));
continue;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName)));
continue;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it.");
return;
}
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
$suspendedServer = null;
if ($terminateStrategy == 'suspend') {
$suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED));
if (count($suspendedServers) > 0) {
$suspendedServer = array_shift($suspendedServers);
}
}
if ($terminateStrategy == 'suspend' && $suspendedServer) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId));
}
if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
} else {
$platform = PlatformFactory::NewPlatform($suspendedServer->platform);
$platform->ResumeServer($suspendedServer);
}
}
}
}
return true;
}
public function StartThread($mysql_farm_role)
{
// Reconfigure observers;
Scalr::ReconfigureObservers();
$db = \Scalr::getDb();
$DBFarmRole = DBFarmRole::LoadByID($mysql_farm_role['id']);
try {
$DBFarm = $DBFarmRole->GetFarmObject();
} catch (Exception $e) {
return;
}
//skip terminated farms
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
return;
}
$tz = Scalr_Environment::init()->loadById($DBFarm->EnvID)->getPlatformConfigValue(ENVIRONMENT_SETTINGS::TIMEZONE);
if ($tz) {
date_default_timezone_set($tz);
}
//
// Check replication status
//
$this->Logger->info("[FarmID: {$DBFarm->ID}] Checking replication status");
$servers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::RUNNING));
//
// Check backups and mysql bandle procedures
//
//Backups
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_ENABLED) && $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) != 0) {
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING) == 1) {
// Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots)
// We must wait for running bundle process.
$bcp_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) * 60 * 5;
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BCP_TS) + $bcp_timeout < time()) {
$bcp_timeouted = true;
}
if ($bcp_timeouted) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING, 0, DBFarmRole::TYPE_LCL);
$this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Backup already running. Timeout. Clear lock.");
}
} else {
$timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BCP_EVERY) * 60;
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BCP_TS) + $timeout < time()) {
$this->Logger->info("[FarmID: {$DBFarm->ID}] Need new backup");
$servers = $DBFarm->GetMySQLInstances(false, true);
if (!$servers[0]) {
$servers = $DBFarm->GetMySQLInstances(true);
} else {
$servers = array_reverse($servers);
}
$DBServer = $servers[0];
if ($DBServer) {
if ($DBServer->status == SERVER_STATUS::RUNNING) {
$msg = new Scalr_Messaging_Msg_Mysql_CreateBackup($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_ROOT_PASSWORD));
$DBServer->SendMessage($msg);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BCP_RUNNING, 1, DBFarmRole::TYPE_LCL);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_BCP_SERVER_ID, $DBServer->serverId, DBFarmRole::TYPE_LCL);
}
} else {
$this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql instances for run backup procedure!");
}
}
}
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_ENABLED) && $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY) != 0) {
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING) == 1) {
// Wait for timeout time * 2 (Example: NIVs problem with big mysql snapshots)
// We must wait for running bundle process.
$bundle_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY) * (3600 * 2);
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BUNDLE_TS) + $bundle_timeout < time()) {
$bundle_timeouted = true;
}
if ($bundle_timeouted) {
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING, 0, DBFarmRole::TYPE_LCL);
$this->Logger->info("[FarmID: {$DBFarm->ID}] MySQL Bundle already running. Timeout. Clear lock.");
}
} else {
/*
* Check bundle window
*/
$bundleEvery = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_EVERY);
$timeout = $bundleEvery * 3600;
$lastBundleTime = $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_LAST_BUNDLE_TS);
$performBundle = false;
if ($bundleEvery % 24 == 0) {
if ($lastBundleTime) {
$days = $bundleEvery / 24;
$bundleDay = (int) date("md", strtotime("+{$days} day", $lastBundleTime));
if ($bundleDay > (int) date("md")) {
return;
}
}
$pbwFrom = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_MM));
$pbwTo = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_MM));
if ($pbwFrom && $pbwTo) {
$current_time = (int) date("Hi");
if ($pbwFrom <= $current_time && $pbwTo >= $current_time) {
$performBundle = true;
}
} else {
$performBundle = true;
}
} else {
//Check timeout
if ($lastBundleTime + $timeout < time()) {
$pbwFrom = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_START_MM));
$pbwTo = (int) ($DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_HH) . $DBFarmRole->GetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_WINDOW_END_MM));
if ($pbwFrom && $pbwTo) {
$current_time = (int) date("Hi");
if ($pbwFrom <= $current_time && $pbwTo >= $current_time) {
$performBundle = true;
}
} else {
$performBundle = true;
}
}
}
if ($performBundle) {
$this->Logger->info("[FarmID: {$DBFarm->ID}] Need mySQL bundle procedure");
// Rebundle
$servers = $DBFarm->GetMySQLInstances(true, false);
$DBServer = $servers[0];
if ($DBServer) {
if ($DBServer->status == SERVER_STATUS::RUNNING) {
$DBServer->SendMessage(new Scalr_Messaging_Msg_Mysql_CreateDataBundle());
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_IS_BUNDLE_RUNNING, 1, DBFarmRole::TYPE_LCL);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_MYSQL_BUNDLE_SERVER_ID, $DBServer->serverId, DBFarmRole::TYPE_LCL);
}
} else {
$this->Logger->info("[FarmID: {$DBFarm->ID}] There is no running mysql master instances for run bundle procedure!");
}
}
}
}
// Check replication
foreach ($servers as $DBServer) {
try {
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) == 1) {
continue;
}
$this->Logger->info("[FarmID: {$DBFarm->ID}] {$DBServer->remoteIp} -> SLAVE STATUS");
$sock = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP);
@socket_set_nonblock($sock);
$time = time();
$res = true;
while (!@socket_connect($sock, $DBServer->remoteIp, 3306)) {
$err = @socket_last_error($sock);
if ($err == 115 || $err == 114 || $err == 36 || $err == 37) {
if (time() - $time >= 5) {
@socket_close($sock);
$res = false;
break;
}
sleep(1);
continue;
} else {
$res = $err == 56 ? true : false;
break;
}
}
if (!$res) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Scalr cannot connect to server %s:3306 (%s) and check replication status. (Error (%s):%s)"), $DBServer->remoteIp, $DBServer->serverId, $err, socket_strerror($err))));
continue;
}
// Connect to Mysql on slave
$conn = NewADOConnection("mysqli");
$conn->Connect($DBServer->remoteIp, 'scalr_stat', $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_MYSQL_STAT_PASSWORD), null);
$conn->SetFetchMode(ADODB_FETCH_ASSOC);
// Get Slave status
$r = $conn->GetRow("SHOW SLAVE STATUS");
// Check slave replication running or not
if ($r['Slave_IO_Running'] == 'Yes' && $r['Slave_SQL_Running'] == 'Yes') {
$replication_status = 1;
} else {
$replication_status = 0;
}
if ($replication_status != $DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_REPLICATION_STATUS)) {
if ($replication_status == 0) {
Scalr::FireEvent($DBFarm->ID, new MySQLReplicationFailEvent($DBServer));
} else {
Scalr::FireEvent($DBFarm->ID, new MySQLReplicationRecoveredEvent($DBServer));
}
}
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, "Cannot retrieve replication status. {$e->getMessage()}"));
}
}
}
