/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$dtNow = new DateTime('now');
try {
$dbServer = DBServer::LoadByID($request->serverId);
} catch (ServerNotFoundException $e) {
$this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage());
return false;
}
if (!in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED])) {
return false;
}
// Check and skip locked instances
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) {
if (($checkDateTime = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME)) <= time()) {
if (!$dbServer->GetRealStatus(true)->isTerminated()) {
$isLocked = $dbServer->GetEnvironmentObject()->aws($dbServer->GetCloudLocation())->ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
if ($isLocked) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Server '%s' has disableAPITermination flag and can't be terminated (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
$startTime = strtotime($dbServer->dateShutdownScheduled);
// 1, 2, 3, 4, 5, 6, 9, 14, ... 60
$diff = round((($checkDateTime < $startTime ? $startTime : $checkDateTime) - $startTime) / 60 * 0.5) * 60;
$diff = $diff == 0 ? 60 : ($diff > 3600 ? 3600 : $diff);
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME, time() + $diff);
return false;
} else {
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isLocked);
}
}
} else {
return false;
}
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) {
try {
$p = PlatformFactory::NewPlatform($dbServer->platform);
$environment = $dbServer->GetEnvironmentObject();
if (!$environment->isPlatformEnabled($dbServer->platform)) {
throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id));
}
if ($dbServer->GetCloudServerID()) {
$serverHistory = $dbServer->getServerHistory();
$isTermination = in_array($dbServer->status, [SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE]);
$isSuspension = in_array($dbServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND]);
/* @var $terminationData TerminationData */
$terminationData = null;
//NOTE: in any case, after call, be sure to set callback to null
$this->setupClientCallback($p, function ($request, $response) use($dbServer, &$terminationData) {
$terminationData = new TerminationData();
$terminationData->serverId = $dbServer->serverId;
if ($request instanceof \http\Client\Request) {
$terminationData->requestUrl = $request->getRequestUrl();
$terminationData->requestQuery = $request->getQuery();
$terminationData->request = $request->toString();
}
if ($response instanceof \http\Client\Response) {
$terminationData->response = $response->toString();
$terminationData->responseCode = $response->getResponseCode();
$terminationData->responseStatus = $response->getResponseStatus();
}
}, $dbServer);
try {
$status = $dbServer->GetRealStatus();
} catch (Exception $e) {
//eliminate callback
$this->setupClientCallback($p, null, $dbServer);
throw $e;
}
//eliminate callback
$this->setupClientCallback($p, null, $dbServer);
if ($dbServer->isCloudstack()) {
//Workaround for when expunge flag not working and servers stuck in Destroyed state.
$isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed';
} else {
$isTerminated = $status->isTerminated();
}
if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) {
try {
if ($dbServer->farmId != 0) {
try {
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]]));
if ($dbServer->index == 1 && $serversCount > 1) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
return false;
}
}
} catch (Exception $e) {
}
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
}
} catch (Exception $e) {
$this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage());
}
$terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME);
if (!$terminationTime || time() - $terminationTime > 180) {
if ($isTermination) {
$p->TerminateServer($dbServer);
} else {
$p->SuspendServer($dbServer);
}
$dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time());
if ($dbServer->farmId) {
$wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbServer);
$event->isSuspended = !$isTermination;
\Scalr::FireEvent($dbServer->farmId, $event);
}
}
}
} else {
if ($dbServer->status == SERVER_STATUS::TERMINATED) {
if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) {
$errorResolution = true;
$serverHistory->setTerminated();
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
}
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) {
$dbServer->updateStatus(SERVER_STATUS::TERMINATED);
$errorResolution = true;
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) {
$dbServer->update(['status' => SERVER_STATUS::SUSPENDED, 'remoteIp' => '', 'localIp' => '']);
$errorResolution = true;
}
}
}
if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) {
//Automatic error resolution
$ste->delete();
}
}
} else {
//If there is no cloudserverID we don't need to add this server into server history.
//$serverHistory->setTerminated();
$dbServer->Remove();
}
} catch (InstanceNotFound $e) {
if ($serverHistory) {
$serverHistory->setTerminated();
}
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
} catch (Exception $e) {
if ($request->serverId && stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false && $dbServer && $dbServer->platform == \SERVER_PLATFORMS::EC2) {
// server has disableApiTermination flag on cloud, update server properties
$dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, 1);
} else {
if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || CloudPlatformSuspensionInfo::isSuspensionException($e) || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) {
//Postpones unsuccessful task for 30 minutes.
$ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage());
$minutes = rand(30, 40);
$ste->retryAfter = new \DateTime('+' . $minutes . ' minutes');
if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
//We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts
$dbServer->Remove();
if (isset($terminationData)) {
$terminationData->save();
}
}
$ste->save();
} else {
$this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage());
throw $e;
}
}
}
}
return $request;
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
$dtNow = new DateTime('now');
try {
$dbServer = DBServer::LoadByID($request->serverId);
} catch (ServerNotFoundException $e) {
$this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage());
return false;
}
if (!in_array($dbServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
return false;
}
//Skip Locked instances
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) {
return false;
}
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (in_array($dbServer->status, array(SERVER_STATUS::TERMINATED)) || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) {
try {
$p = PlatformFactory::NewPlatform($dbServer->platform);
$environment = $dbServer->GetEnvironmentObject();
if (!$environment->isPlatformEnabled($dbServer->platform)) {
throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id));
}
if ($dbServer->GetCloudServerID()) {
$serverHistory = $dbServer->getServerHistory();
$isTermination = in_array($dbServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$isSuspension = in_array($dbServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND));
$status = $dbServer->GetRealStatus();
if ($dbServer->isCloudstack()) {
//Workaround for when expunge flag not working and servers stuck in Destroyed state.
$isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed';
} else {
$isTerminated = $status->isTerminated();
}
if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) {
try {
if ($dbServer->farmId != 0) {
try {
if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]]));
if ($dbServer->index == 1 && $serversCount > 1) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. " . "Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
return false;
}
}
} catch (Exception $e) {
}
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId));
}
} catch (Exception $e) {
$this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage());
}
$terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME);
if (!$terminationTime || time() - $terminationTime > 180) {
if ($isTermination) {
$p->TerminateServer($dbServer);
} else {
$p->SuspendServer($dbServer);
}
$dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time());
if ($dbServer->farmId) {
$wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown'));
if (!$wasHostDownFired) {
$event = new HostDownEvent($dbServer);
$event->isSuspended = !$isTermination;
\Scalr::FireEvent($dbServer->farmId, $event);
}
}
}
} else {
if ($dbServer->status == SERVER_STATUS::TERMINATED) {
if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) {
$errorResolution = true;
$serverHistory->setTerminated();
$dbServer->Remove();
}
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) {
$dbServer->status = SERVER_STATUS::TERMINATED;
$dbServer->Save();
$errorResolution = true;
} else {
if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) {
$dbServer->status = SERVER_STATUS::SUSPENDED;
$dbServer->remoteIp = '';
$dbServer->localIp = '';
$dbServer->Save();
$errorResolution = true;
}
}
}
if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) {
//Automatic error resolution
$ste->delete();
}
}
} else {
//If there is no cloudserverID we don't need to add this server into server history.
//$serverHistory->setTerminated();
$dbServer->Remove();
}
} catch (InstanceNotFound $e) {
if ($serverHistory) {
$serverHistory->setTerminated();
}
$dbServer->Remove();
} catch (Exception $e) {
if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || stripos($e->getMessage(), "The request you have made requires authentication.") !== false || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) {
//Postpones unsuccessful task for 30 minutes.
$ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage());
$minutes = rand(30, 40);
$ste->retryAfter = new \DateTime('+' . $minutes . ' minutes');
if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) {
//We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts
$dbServer->Remove();
}
$ste->save();
} else {
$this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage());
throw $e;
}
}
}
return $request;
}
